diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 6b4a85f..644721c 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -74,8 +74,10 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: "CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake SKBUILD_CMAKE_ARGS=-DSDE_WARNINGS_AS_ERRORS=ON" # Bundle openblas.dll into the wheel (auditwheel/delocate handle this on Linux/macOS) CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel} --add-path C:/vcpkg/installed/x64-windows/bin" - # On free-threaded builds, assert the import did not re-enable the GIL. - CIBW_TEST_COMMAND: python -c "import sys, sysconfig, sparsediffpy; ft = bool(sysconfig.get_config_var('Py_GIL_DISABLED')); assert not ft or not sys._is_gil_enabled(), 'GIL re-enabled on import'; print('freethreaded_build=', ft)" + # On free-threaded builds, assert the import did not re-enable the GIL, + # then run the test suite against the installed wheel. + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: python -c "import sys, sysconfig, sparsediffpy; ft = bool(sysconfig.get_config_var('Py_GIL_DISABLED')); assert not ft or not sys._is_gil_enabled(), 'GIL re-enabled on import'; print('freethreaded_build=', ft)" && pytest {project}/tests -q - name: Check wheels shell: bash run: | diff --git a/README.md b/README.md index 3a02ca6..87a77bb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # SparseDiffPy -Python bindings for [SparseDiffEngine](https://github.com/SparseDifferentiation/SparseDiffEngine), a C library for computing sparse Jacobians and Hessians. +Sparse Jacobians and Hessians of nonlinear expressions, computed by +[SparseDiffEngine](https://github.com/SparseDifferentiation/SparseDiffEngine), +a C library that works out the sparsity pattern once and then only computes +the structural nonzeros. Results are `scipy.sparse.csr_matrix`. ## Installation @@ -8,12 +11,111 @@ Python bindings for [SparseDiffEngine](https://github.com/SparseDifferentiation/ pip install sparsediffpy ``` -## Usage +From source: `git clone --recursive https://github.com/SparseDifferentiation/SparseDiffPy && pip install -e SparseDiffPy`. + +## Quick start + +```python +import numpy as np +import sparsediffpy as sp + +scope = sp.Scope() # owns the flat variable vector +x = scope.Variable(3, 1) # shapes are always 2-D: (rows, cols) + +f = sp.exp(x) + 2.0 * x # build an expression with operators and sp.* atoms +fn = sp.compile(f) # analyze sparsity once + +x.value = [1.0, 2.0, 3.0] +print(fn.forward()) # [[4.718] [11.389] [26.086]], shaped like f +print(fn.jacobian().toarray()) # diag(exp(x) + 2), a 3x3 csr_matrix +print(fn.hessian(np.ones(3))) # Hessian of w' f(x) for weights w +``` + +`forward()` has the shape of `f`, `jacobian()` has shape `(f.size, n_vars)` +and `hessian(w)` has shape `(n_vars, n_vars)`, where `n_vars` is the total +size of all variables in the scope, in declaration order. Values must have +the exact shape of what they set: `x.value` takes an array shaped like `x`, +`hessian(w)` takes `w` shaped like `f` (for vectors a 1-D array also works), +and flat vectors such as `scope.set_values(u)` must be 1-D. Flat vectors (the +rows of the Jacobian, the scope values) are column-major. + +**Parameters** are data you can change without recompiling: ```python -from sparsediffpy import _sparsediffengine +import numpy as np +import sparsediffpy as sp + +scope = sp.Scope() +x = scope.Variable(3, 1) +A = scope.Parameter(2, 3, value=np.ones((2, 3))) +fn = sp.compile(sp.sin(A @ x)) + +x.value = [0.1, 0.2, 0.3] +J1 = fn.jacobian() +A.value = np.arange(6.0).reshape(2, 3) # no recompile needed +J2 = fn.jacobian() ``` +## Using it in a solver + +`sp.Problem(objective, constraints)` exposes the callbacks NLP solvers ask for: +objective value and gradient, constraint values and Jacobian, and the Lagrangian +Hessian `obj_factor * H_f + sum_i lagrange_i * H_gi`. A stage runs in the engine +only when its point changed: a forward called again with the same `u` (or with +unchanged scope values and parameters) returns the cached result, and +`gradient()`/`jacobian()` are reused until the matching forward moves. Returned +arrays are owned by the problem, so do not modify them in place. Here it drives SciPy: + +```python +import numpy as np +from scipy.optimize import NonlinearConstraint, minimize + +import sparsediffpy as sp + +# minimize sum(exp(x) - c * x) subject to ||x||^2 <= 1 +scope = sp.Scope() +x = scope.Variable(4, 1) +c = np.array([[1.0], [2.0], [3.0], [4.0]]) # constants are 2-D, like variables +prob = sp.Problem(sp.sum(sp.exp(x) - c * x), [sp.sum(x ** 2)]) + +def grad(u): + prob.objective_forward(u) + return prob.gradient() + +def obj_hess(u): # the Lagrangian Hessian needs both forwards at u (cached if done) + prob.objective_forward(u); prob.constraint_forward(u) + return prob.hessian(1.0, [0.0]) + +def con_jac(u): + prob.constraint_forward(u) + return prob.jacobian() + +def con_hess(u, v): + prob.objective_forward(u); prob.constraint_forward(u) + return prob.hessian(0.0, v) + +con = NonlinearConstraint(prob.constraint_forward, -np.inf, 1.0, + jac=con_jac, hess=con_hess) +res = minimize(prob.objective_forward, np.zeros(4), jac=grad, hess=obj_hess, + method="trust-constr", constraints=con) +print(res.x) # approx. [0. 0.289 0.549 0.784], on the boundary ||x|| = 1 +``` + +## Supported operations + +- **Operators:** `+ - * @ **`, `/` by a constant, `.T`, and indexing with ints, slices and integer lists: `x[1:3]` on a + vector, `X[0, :]` or `X[[0, 2], 1]` on a matrix (a matrix always takes two indices). + Scalars, rows and columns broadcast. NumPy arrays and `scipy.sparse` matrices are accepted as + constants; like every expression they must be 2-D, so pass `a[:, None]` or `a[None, :]` for a 1-D `a`. +- **Elementwise:** `sin cos tan exp log sinh tanh asinh atan atanh logistic normal_cdf entr xexp power atan2`. +- **Affine:** `sum prod reshape broadcast hstack vstack trace diag_vec index_flat`. +- **Other:** `quad_form quad_over_lin rel_entr`. + +`log`, `tan`, `atanh`, `entr`, `atan2`, `rel_entr`, `quad_over_lin` and `prod` +currently need plain Variables as arguments (an engine limitation). + +The low-level C bindings remain available as `sparsediffpy._sparsediffengine`. + ## Free-threaded Python Wheels are published for free-threaded CPython 3.14 (`cp314t`) as well as the diff --git a/conda.recipe/recipe.yaml b/conda.recipe/recipe.yaml index 70394b3..47b66da 100644 --- a/conda.recipe/recipe.yaml +++ b/conda.recipe/recipe.yaml @@ -56,6 +56,7 @@ requirements: run: - python - numpy >=2 + - scipy tests: - python: diff --git a/pyproject.toml b/pyproject.toml index 20133b0..4c87286 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "sparsediffpy" version = "0.9.0" description = "Python bindings for SparseDiffEngine algorithmic differentiation" requires-python = ">=3.11" -dependencies = ["numpy >= 2.0.0"] +dependencies = ["numpy >= 2.0.0", "scipy"] license = "Apache-2.0" [project.readme] diff --git a/sparsediffpy/__init__.py b/sparsediffpy/__init__.py index 9b73c36..08995f9 100644 --- a/sparsediffpy/__init__.py +++ b/sparsediffpy/__init__.py @@ -15,3 +15,12 @@ """ from sparsediffpy import _sparsediffengine # noqa: F401 +from sparsediffpy._expression import ( # noqa: F401 + DimensionError, Expression, Parameter, Scope, Variable, broadcast, index_flat, +) +from sparsediffpy._atoms import ( # noqa: F401 + asinh, atan, atan2, atanh, cos, diag_vec, entr, exp, hstack, log, logistic, + normal_cdf, power, prod, quad_form, quad_over_lin, rel_entr, reshape, sin, + sinh, sum, tan, tanh, trace, vstack, xexp, +) +from sparsediffpy._compile import CompiledExpression, Problem, compile # noqa: F401 diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/_atoms.py new file mode 100644 index 0000000..58ce53e --- /dev/null +++ b/sparsediffpy/_atoms.py @@ -0,0 +1,189 @@ +"""Named functions: elementwise atoms, reductions, stacking, and special forms.""" + +import numpy as np +import scipy.sparse as sps + +from sparsediffpy import _sparsediffengine as _C +from sparsediffpy._expression import ( + Constant, Node, Parameter, Variable, _check_shape, _check_symmetric, + cast_to_const, +) + + +def _require_variables(name, *xs): + """Restricted-domain atoms in the engine only accept plain Variables.""" + if not all(isinstance(x, Variable) for x in xs): + raise ValueError(f"{name} cannot be applied directly to a composite " + "expression; its arguments must be plain Variables") + if len(xs) == 2 and xs[0] is xs[1]: + raise ValueError(f"{name}: arguments must be distinct Variables") + + +def _elementwise(make, name, restricted=False): + """Make the elementwise atom ``name`` from its engine constructor. + + A restricted atom (domain not all of R) only accepts a plain Variable. + """ + def atom(x): + x = cast_to_const(x) + if restricted: + _require_variables(name, x) + return Node(make, [x], x.shape) + atom.__name__ = atom.__qualname__ = name + atom.__doc__ = f"Elementwise {name}." + return atom + + +sin = _elementwise(_C.make_sin, "sin") +cos = _elementwise(_C.make_cos, "cos") +tan = _elementwise(_C.make_tan, "tan", restricted=True) +exp = _elementwise(_C.make_exp, "exp") +log = _elementwise(_C.make_log, "log", restricted=True) +sinh = _elementwise(_C.make_sinh, "sinh") +tanh = _elementwise(_C.make_tanh, "tanh") +asinh = _elementwise(_C.make_asinh, "asinh") +atanh = _elementwise(_C.make_atanh, "atanh", restricted=True) +atan = _elementwise(_C.make_atan, "atan") +logistic = _elementwise(_C.make_logistic, "logistic") +normal_cdf = _elementwise(_C.make_normal_cdf, "normal_cdf") +entr = _elementwise(_C.make_entr, "entr", restricted=True) +xexp = _elementwise(_C.make_xexp, "xexp") + + +def power(x, p): + """Elementwise x ** p for a constant exponent p.""" + return cast_to_const(x) ** p + + +def atan2(y, x): + """Elementwise atan2(y, x) for distinct Variables of the same shape.""" + _require_variables("atan2", y, x) + if y.shape != x.shape: + raise ValueError(f"atan2: shape mismatch {y.shape} vs {x.shape}") + return Node(_C.make_atan2, [y, x], y.shape) + + +def _reduction_shape(x, axis): + """Output shape of sum/prod. The engine always returns a row vector.""" + d1, d2 = x.shape + shapes = {None: (1, 1), 0: (1, d2), 1: (1, d1)} + if axis not in shapes: + raise ValueError(f"Invalid axis {axis}, must be None, 0, or 1") + return shapes[axis] + + +def sum(x, axis=None): + """Sum of entries: axis=None -> (1, 1), axis=0 -> (1, d2), axis=1 -> (1, d1).""" + x = cast_to_const(x) + shape = _reduction_shape(x, axis) + c_axis = -1 if axis is None else axis + return Node(lambda c: _C.make_sum(c, c_axis), [x], shape) + + +def prod(x, axis=None): + """Product of entries; shapes as for :func:`sum`. ``x`` must be a Variable.""" + if not isinstance(x, Variable): + raise ValueError("prod requires its argument to be a plain Variable " + "(the engine has no chain rule for prod yet)") + shape = _reduction_shape(x, axis) + make = {None: _C.make_prod, 0: _C.make_prod_axis_zero, + 1: _C.make_prod_axis_one}[axis] + return Node(make, [x], shape) + + +def reshape(x, d1, d2): + """Column-major reshape.""" + x, shape = cast_to_const(x), _check_shape(d1, d2) + if x.size != shape[0] * shape[1]: + raise ValueError(f"Cannot reshape {x.shape} to {shape}") + return Node(lambda c: _C.make_reshape(c, *shape), [x], shape) + + +def trace(x): + """Sum of the diagonal of a square matrix, as a (1, 1) expression.""" + x = cast_to_const(x) + if x.shape[0] != x.shape[1]: + raise ValueError(f"trace requires a square matrix, got {x.shape}") + return Node(_C.make_trace, [x], (1, 1)) + + +def diag_vec(x): + """(n, 1) column vector -> (n, n) diagonal matrix.""" + x = cast_to_const(x) + if x.shape[1] != 1: + raise ValueError(f"diag_vec requires a column vector, got {x.shape}") + return Node(_C.make_diag_vec, [x], (x.shape[0], x.shape[0])) + + +def hstack(exprs): + """Concatenate horizontally; all arguments need the same number of rows.""" + exprs = [cast_to_const(e) for e in exprs] + if not exprs or len({e.shape[0] for e in exprs}) != 1: + raise ValueError(f"hstack: need >=1 args with equal rows, got " + f"{[e.shape for e in exprs]}") + shape = (exprs[0].shape[0], np.sum([e.shape[1] for e in exprs]).item()) + return Node(lambda *caps: _C.make_hstack(list(caps)), exprs, shape) + + +def vstack(exprs): + """Concatenate vertically; all arguments need the same number of columns.""" + exprs = [cast_to_const(e) for e in exprs] + if not exprs or len({e.shape[1] for e in exprs}) != 1: + raise ValueError(f"vstack: need >=1 args with equal columns, got " + f"{[e.shape for e in exprs]}") + shape = (np.sum([e.shape[0] for e in exprs]).item(), exprs[0].shape[1]) + return Node(lambda *caps: _C.make_vstack(list(caps)), exprs, shape) + + +def quad_form(x, Q): + """x' Q x for a column vector x (n, 1) and a constant or Parameter Q (n, n). + + A NumPy Q uses the engine's dense (BLAS) path, a scipy.sparse Q the sparse + path. Constant Q is symmetrized; a Parameter Q must be symmetric, which is + checked here and on every later ``Q.value`` update. + """ + x, Q = cast_to_const(x), cast_to_const(Q) + n = x.shape[0] + if x.shape[1] != 1 or Q.shape != (n, n): + raise ValueError(f"quad_form: need x (n, 1) and Q (n, n), " + f"got {x.shape} and {Q.shape}") + if isinstance(Q, Parameter): + Q._symmetric = True # checked now and on every later Q.value update + if Q._flat is not None: + _check_symmetric(Q._flat, Q.shape) + return Node(lambda p, c: _C.make_quad_form(p, c, "dense", None, n), + [Q, x], (1, 1)) + if not isinstance(Q, Constant): + raise TypeError("quad_form: Q must be a constant or a Parameter") + V = (Q.value + Q.value.T) / 2 # the engine assumes symmetric Q + if not sps.issparse(V): + data = np.ascontiguousarray(V).ravel() + return Node(lambda c: _C.make_quad_form(None, c, "dense", data, n), + [x], (1, 1)) + V = sps.csr_matrix(V) + return Node(lambda c: _C.make_quad_form( + None, c, "sparse", V.data, V.indices.astype(np.int32), + V.indptr.astype(np.int32), n, n), [x], (1, 1)) + + +def quad_over_lin(x, z): + """sum(x**2) / z for distinct Variables x and scalar z.""" + _require_variables("quad_over_lin", x, z) + if z.shape != (1, 1): + raise ValueError(f"quad_over_lin: z must be scalar, got {z.shape}") + return Node(_C.make_quad_over_lin, [x, z], (1, 1)) + + +def rel_entr(x, y): + """Elementwise x * log(x / y) for distinct Variables x, y. + + Shapes must match, or one of them must be scalar. + """ + _require_variables("rel_entr", x, y) + if x.shape == y.shape or y.shape == (1, 1): + shape = x.shape + elif x.shape == (1, 1): + shape = y.shape + else: + raise ValueError(f"rel_entr: incompatible shapes {x.shape} and {y.shape}") + return Node(_C.make_rel_entr, [x, y], shape) diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py new file mode 100644 index 0000000..37f25eb --- /dev/null +++ b/sparsediffpy/_compile.py @@ -0,0 +1,244 @@ +"""Turn expressions into an engine problem and evaluate it. + +``compile(f)`` is a thin wrapper around ``Problem``: f becomes the only +constraint of a problem with a zero objective, so the engine's constraint +Jacobian is J_f and its Lagrangian Hessian (objective weight 0) is the Hessian +of w' f. +""" + +import numpy as np +import scipy.sparse as sps + +from sparsediffpy import _sparsediffengine as _C +from sparsediffpy._expression import ( + Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const, +) + + +def _find_scope(exprs): + """Return the single Scope shared by all Variables/Parameters in exprs.""" + scopes, has_var, seen, stack = set(), False, set(), list(exprs) + while stack: + e = stack.pop() + if id(e) in seen: + continue + seen.add(id(e)) + if isinstance(e, (Variable, Parameter)): + scopes.add(e._scope) + has_var |= isinstance(e, Variable) + stack.extend(e.args) + if not has_var: + raise ValueError("Expression must contain at least one Variable") + if len(scopes) > 1: + raise ValueError("All variables must belong to the same Scope") + return scopes.pop() + + +class _Builder: + """Builds C capsules bottom-up, sharing repeated subexpressions. + + The walk uses an explicit stack rather than recursion, so deep expressions + (e.g. a long sum built in a Python loop) do not hit Python's recursion + limit. Nodes are built in the same order as a recursive left-to-right + post-order traversal. + """ + + def __init__(self, scope): + self.n_vars = scope.total_var_size + self.cache = {} + self.params = [] # (Parameter, capsule), each once + + def __call__(self, root): + """Return root's capsule, building any children not built yet.""" + stack = [root] + while stack: + e = stack[-1] + if id(e) in self.cache: + stack.pop() + continue + pending = [a for a in e.args if id(a) not in self.cache] + if pending: + stack.extend(reversed(pending)) # build the leftmost child first + continue + stack.pop() + self.cache[id(e)] = self._build_one(e) + return self.cache[id(root)] + + def _build_one(self, e): + """Build e's capsule; all of e's children must already be cached.""" + d1, d2 = e.shape + if isinstance(e, Variable): + cap = _C.make_variable(d1, d2, e._offset, self.n_vars) + elif isinstance(e, Parameter): + values = e._flat if e._flat is not None else np.zeros(e.size) + cap = _C.make_parameter(d1, d2, e._offset, self.n_vars, values) + self.params.append((e, cap)) + elif isinstance(e, Constant): + values = e.dense().ravel(order="F") + cap = _C.make_parameter(d1, d2, -1, self.n_vars, values) + else: + cap = e.make(*[self.cache[id(a)] for a in e.args]) + if _C.get_expr_dimensions(cap) != e.shape: + raise ValueError(f"Engine shape {_C.get_expr_dimensions(cap)} != " + f"expected {e.shape}") + return cap + + +def _csr(triple): + """scipy CSR matrix from an engine (data, indices, indptr, shape) tuple.""" + data, indices, indptr, shape = triple + return sps.csr_matrix((data, indices, indptr), shape=shape) + + +class Problem: + """A scalar objective plus a list of constraint expressions. + + Typical solver loop (u is the flat variable vector):: + + prob.objective_forward(u); prob.constraint_forward(u) + prob.gradient(); prob.jacobian(); prob.hessian(obj_factor, lagrange) + + Each stage runs in the engine only when its point changed: a forward is + skipped when u (or the scope's values and parameters) is unchanged, and + gradient/jacobian are skipped when the matching forward has not moved. + Skipped calls return the cached object, which the problem owns: do not + modify it in place. + """ + + def __init__(self, objective, constraints=(), verbose=False): + objective = cast_to_const(objective) + constraints = [cast_to_const(c) for c in constraints] + if objective.shape != (1, 1): + raise ValueError(f"Objective must be scalar, got {objective.shape}") + self._scope = _find_scope([objective, *constraints]) + # The objective and the constraints are evaluated at separate points, so + # they must not share engine nodes: a constraint_forward would overwrite + # values that the objective's gradient still reads. + obj_build, con_build = _Builder(self._scope), _Builder(self._scope) + obj_cap = obj_build(objective) + con_caps = [con_build(c) for c in constraints] + self._capsule = _C.make_problem(obj_cap, con_caps, verbose) + params = obj_build.params + con_build.params + self._params = [p for p, _ in params] + if params: + _C.problem_register_params(self._capsule, [c for _, c in params]) + _C.problem_init_derivatives(self._capsule) + self._param_version = None + self.n_vars = self._scope.total_var_size + self.total_constraint_size = sum(c.size for c in constraints) + # Point keys: ("scope", values_version, param_version) or ("u", u_version, + # param_version). Each stage remembers the key it was last computed at. + self._last_u, self._u_version = None, 0 + self._obj_at = self._con_at = self._grad_at = self._jac_at = None + + def _sync_params(self): + """Push Parameter values to the engine if any changed since last time.""" + if not self._params or self._param_version == self._scope._param_version: + return + theta = np.zeros(self._scope._n_params) + for p in self._params: + if p._flat is None: + raise ValueError(f"Parameter with shape {p.shape} has no value; " + "set parameter.value before evaluating") + theta[p._offset:p._offset + p.size] = p._flat + _C.problem_update_params(self._capsule, theta) + self._param_version = self._scope._param_version + + def _point(self, u): + """Return (key, flat u) for the evaluation point u (None: scope values). + + The C side does not check the length of u, so it is checked here. + """ + if u is None: + key = ("scope", self._scope._values_version) + u = self._scope._values[:self.n_vars] + else: + u = _flatten_vector(u, self.n_vars, "u") + if self._last_u is None or not np.array_equal(u, self._last_u): + self._last_u, self._u_version = u.copy(), self._u_version + 1 + key = ("u", self._u_version) + return key + (self._scope._param_version,), u + + def objective_forward(self, u=None): + """Objective value at u (default: the scope's current values).""" + key, u = self._point(u) + if self._obj_at != key: + self._sync_params() + self._obj_value = _C.problem_objective_forward(self._capsule, u) + self._obj_at = key + return self._obj_value + + def constraint_forward(self, u=None): + """Stacked constraint values at u (default: the scope's current values).""" + key, u = self._point(u) + if self._con_at != key: + self._sync_params() + self._con_value = _C.problem_constraint_forward(self._capsule, u) + self._con_at = key + return self._con_value + + def gradient(self): + """Objective gradient at the point of the last objective_forward.""" + if self._obj_at is None: + raise RuntimeError("call objective_forward before gradient") + if self._grad_at != self._obj_at: + self._grad_value = _C.problem_gradient(self._capsule) + self._grad_at = self._obj_at + return self._grad_value + + def jacobian(self): + """Constraint Jacobian (csr_matrix) at the last constraint_forward point.""" + if self._con_at is None: + raise RuntimeError("call constraint_forward before jacobian") + if self._jac_at != self._con_at: + self._jac_value = _csr(_C.problem_jacobian(self._capsule)) + self._jac_at = self._con_at + return self._jac_value + + def hessian(self, obj_factor, lagrange): + """Lagrangian Hessian obj_factor H_f + sum_i lagrange_i H_gi (csr_matrix). + + Call objective_forward and constraint_forward at the same point first. + """ + lagrange = _flatten_vector(lagrange, self.total_constraint_size, "lagrange") + # The engine's Hessian reads Jacobian values cached by eval_jacobian, so + # the gradient and Jacobian must be current (no-ops if they already are). + self.gradient() + if self.total_constraint_size: + self.jacobian() + if self._con_at != self._obj_at: + raise RuntimeError("objective_forward and constraint_forward must " + "be evaluated at the same point before hessian") + return _csr(_C.problem_hessian(self._capsule, float(obj_factor), lagrange)) + + +class CompiledExpression: + """A compiled expression f, evaluated at the scope's current values.""" + + def __init__(self, expr): + self.shape = expr.shape + self._problem = Problem(Constant(0.0), [expr]) + + def forward(self): + """f(x) as an array of shape f.shape.""" + return self._problem.constraint_forward().reshape(self.shape, order="F") + + def jacobian(self): + """Sparse Jacobian df/dx, shape (f.size, n_vars).""" + self.forward() + return self._problem.jacobian() + + def hessian(self, weights): + """Sparse Hessian of sum(weights * f(x)), shape (n_vars, n_vars). + + weights has f's shape (a 1-D array is also accepted when f is a vector). + """ + weights = _flatten_shaped(weights, self.shape) + self._problem.objective_forward() # the zero objective; needed by gradient() + self.forward() + return self._problem.hessian(0.0, weights) + + +def compile(expr): + """Analyze the sparsity of ``expr`` once and return a CompiledExpression.""" + return CompiledExpression(cast_to_const(expr)) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py new file mode 100644 index 0000000..6cecaf4 --- /dev/null +++ b/sparsediffpy/_expression.py @@ -0,0 +1,379 @@ +"""Expression graph: Scope, leaves (Variable/Parameter/Constant), and operators. + +Every non-leaf is a generic ``Node(make, args, shape)``: ``make`` receives the +C capsules of ``args`` and returns the node's own capsule. Shapes are always +2-D ``(d1, d2)`` and values are flattened column-major, like the C engine. +""" + +import numbers + +import numpy as np +import scipy.sparse as sps + +from sparsediffpy import _sparsediffengine as _C + + +class DimensionError(ValueError): + """Raised when a value has the wrong number of elements.""" + + +def _check_shape(d1, d2): + """Return (d1, d2) as a tuple of positive Python ints, or raise.""" + if not all(isinstance(d, (int, np.integer)) for d in (d1, d2)): + raise TypeError(f"Shape dimensions must be integers, got ({d1!r}, {d2!r})") + if d1 <= 0 or d2 <= 0: + raise ValueError(f"Shape dimensions must be positive, got ({d1}, {d2})") + return (int(d1), int(d2)) + + +class Expression: + """Base class for all expressions. Supports + - * @ ** [] and .T.""" + + __array_ufunc__ = None # make NumPy defer to our reflected operators + shape = None + args = () + + @property + def size(self): + """Number of entries, d1 * d2.""" + return self.shape[0] * self.shape[1] + + @property + def T(self): + """Transpose.""" + return Node(_C.make_transpose, [self], self.shape[::-1]) + + def __add__(self, other): + return _add(self, cast_to_const(other)) + + def __radd__(self, other): + return _add(cast_to_const(other), self) + + def __sub__(self, other): + return _add(self, -cast_to_const(other)) + + def __rsub__(self, other): + return _add(cast_to_const(other), -self) + + def __neg__(self): + return Node(_C.make_neg, [self], self.shape) + + def __mul__(self, other): + return _mul(self, cast_to_const(other)) + + def __rmul__(self, other): + return _mul(cast_to_const(other), self) + + def __truediv__(self, other): + """self / c for a constant c (a number or array), as self * (1 / c).""" + if isinstance(other, Expression) or sps.issparse(other): + raise TypeError("Can only divide by a constant number or dense array; " + "for an expression d use x * d ** -1") + c = np.asarray(other, dtype=np.float64) + if np.any(c == 0): + raise ZeroDivisionError("division of an expression by zero") + return self * (1.0 / c) + + def __matmul__(self, other): + return _matmul(self, cast_to_const(other)) + + def __rmatmul__(self, other): + return _matmul(cast_to_const(other), self) + + def __pow__(self, p): + if not isinstance(p, numbers.Real): + raise TypeError("Exponent must be a constant number") + return Node(lambda c: _C.make_power(c, float(p)), [self], self.shape) + + def __getitem__(self, key): + return _index(self, key) + + def __iter__(self): + # Without this, Python would iterate via __getitem__ until IndexError. + raise TypeError("Expressions are not iterable; index with x[i, j] or " + "pass a list of expressions") + + +class Node(Expression): + """An operation: ``make(*child_capsules)`` builds its C capsule.""" + + def __init__(self, make, args, shape): + self.make = make + self.args = tuple(args) + self.shape = tuple(shape) + + +class Constant(Expression): + """Fixed dense or scipy.sparse data.""" + + def __init__(self, value): + if value is None: + raise TypeError("Constant value cannot be None") + if sps.issparse(value): + self.value = sps.csr_matrix(value, dtype=np.float64) + else: + value = np.asarray(value, dtype=np.float64) + if value.size == 1: + value = value.reshape(1, 1) + elif value.ndim != 2: + raise ValueError( + f"Constants must be scalars or 2-D arrays, (n, 1) or (1, n); " + f"got shape {value.shape}. Use a[:, None] or a[None, :].") + self.value = value + self.shape = _check_shape(*self.value.shape) + + def dense(self): + """The value as a dense array.""" + return self.value.toarray() if sps.issparse(self.value) else self.value + + +class Variable(Expression): + """A block of the scope's flat variable vector. Create via Scope.Variable.""" + + def __init__(self, scope, offset, shape): + self._scope, self._offset, self.shape = scope, offset, shape + + @property + def value(self): + """The variable's current entries, shaped like the variable.""" + flat = self._scope._values[self._offset:self._offset + self.size] + return flat.reshape(self.shape, order="F").copy() + + @value.setter + def value(self, val): + self._scope._values[self._offset:self._offset + self.size] = \ + _flatten_shaped(val, self.shape) + self._scope._values_version += 1 + + +class Parameter(Expression): + """Data that can change without recompiling. Create via Scope.Parameter.""" + + def __init__(self, scope, offset, shape, value=None): + self._scope, self._offset, self.shape = scope, offset, shape + self._flat = None + self._symmetric = False # set by quad_form; the engine assumes Q = Q' + if value is not None: + self.value = value + + @property + def value(self): + """The parameter's current value, or None if it was never set.""" + if self._flat is None: + return None + return self._flat.reshape(self.shape, order="F").copy() + + @value.setter + def value(self, val): + flat = _flatten_shaped(val, self.shape) + if self._symmetric: + _check_symmetric(flat, self.shape) + self._flat = flat + self._scope._param_version += 1 + + +def _check_symmetric(flat, shape): + """Raise unless the column-major ``flat`` is a symmetric matrix of ``shape``.""" + M = flat.reshape(shape, order="F") + if not np.allclose(M, M.T): + raise ValueError("Parameter used in quad_form must be symmetric") + + +def _flatten_shaped(val, shape): + """Flatten a value of shape ``shape`` column-major. + + Also accepts a 1-D array for a vector shape (n, 1) or (1, n), and a scalar + for (1, 1). Any other shape raises, even with the right element count. + """ + if val is None: + raise TypeError("value cannot be None") + arr = np.asarray(val, dtype=np.float64) + if not (arr.shape == shape + or (arr.ndim == 0 and shape == (1, 1)) + or (arr.ndim == 1 and min(shape) == 1 + and arr.size == shape[0] * shape[1])): + raise DimensionError(f"expected shape {shape}, got {arr.shape}") + return arr.ravel(order="F") + + +def _flatten_vector(val, n, name): + """Return ``val`` as a float64 1-D array of exactly n entries.""" + arr = np.asarray(val, dtype=np.float64) + if arr.ndim != 1 or arr.size != n: + raise DimensionError(f"{name} must be a 1-D array of {n} elements, " + f"got shape {arr.shape}") + return arr + + +class Scope: + """Owns the flat variable vector and the parameter vector.""" + + def __init__(self): + self._values = np.zeros(0) + self._values_version = 0 # bumped on every write; problems skip stale work + self._n_params = 0 + self._param_version = 0 + + def Variable(self, d1, d2): + """Create a (d1, d2) variable, appending its entries to the flat vector.""" + shape = _check_shape(d1, d2) + var = Variable(self, self._values.size, shape) + self._values = np.concatenate([self._values, np.zeros(var.size)]) + return var + + def Parameter(self, d1, d2, value=None): + """Create a (d1, d2) parameter, optionally with an initial value.""" + shape = _check_shape(d1, d2) + param = Parameter(self, self._n_params, shape, value) + self._n_params += param.size + return param + + @property + def total_var_size(self): + """Length of the flat variable vector.""" + return self._values.size + + def set_values(self, array): + """Set all variables at once from a flat array (declaration order).""" + self._values[:] = _flatten_vector(array, self._values.size, "values") + self._values_version += 1 + + def get_values(self): + """A copy of the flat variable vector (declaration order).""" + return self._values.copy() + + +def cast_to_const(value): + """Return ``value`` as an Expression, wrapping non-Expressions in a Constant.""" + return value if isinstance(value, Expression) else Constant(value) + + +def _is_data(e): + """Whether e is data (Constant or Parameter) rather than a function of x.""" + return isinstance(e, (Constant, Parameter)) + + +def _broadcast_shape(a, b): + """Shape of a op b when scalars, rows and columns broadcast, or raise.""" + out = [] + for da, db in zip(a, b): + if da != db and 1 not in (da, db): + raise ValueError(f"Cannot broadcast shapes {a} and {b}") + out.append(max(da, db)) + return tuple(out) + + +def broadcast(x, shape): + """Broadcast a scalar, row or column expression to ``shape``.""" + x, shape = cast_to_const(x), _check_shape(*shape) + if x.shape == shape: + return x + if _broadcast_shape(x.shape, shape) != shape: + raise ValueError(f"Cannot broadcast {x.shape} to {shape}") + if isinstance(x, Constant): + return Constant(np.broadcast_to(x.dense(), shape)) + return Node(lambda c: _C.make_broadcast(c, *shape), [x], shape) + + +def _add(a, b): + """a + b, broadcasting both operands to the common shape first.""" + shape = _broadcast_shape(a.shape, b.shape) + return Node(_C.make_add, [broadcast(a, shape), broadcast(b, shape)], shape) + + +def _mul(a, b): + """Elementwise a * b. + + A data operand (constant or parameter) uses the engine's scalar or vector + scaling atoms; two functions of x use the general product. + """ + if _is_data(b) and not _is_data(a): + a, b = b, a # put the constant/parameter operand (if any) first + if _is_data(a) and a.shape == (1, 1): + return Node(_C.make_param_scalar_mult, [a, b], b.shape) + shape = _broadcast_shape(a.shape, b.shape) + a, b = broadcast(a, shape), broadcast(b, shape) + make = _C.make_param_vector_mult if _is_data(a) else _C.make_multiply + return Node(make, [a, b], shape) + + +def _matmul(a, b): + """a @ b: data times a function of x (either side), or two functions of x.""" + if a.shape[1] != b.shape[0]: + raise ValueError(f"Matmul shape mismatch: {a.shape} @ {b.shape}") + shape = (a.shape[0], b.shape[1]) + if _is_data(a): + return Node(_matmul_with_data(_C.make_left_matmul, a), _data_args(a, b), + shape) + if _is_data(b): + return Node(_matmul_with_data(_C.make_right_matmul, b), _data_args(b, a), + shape) + return Node(_C.make_matmul, [a, b], shape) + + +def _data_args(data, x): + """Children of a data matmul: a Parameter is a child, a Constant is baked in.""" + return [data, x] if isinstance(data, Parameter) else [x] + + +def _matmul_with_data(make, A): + """Return make(child) / make(param, child) for a constant/parameter A.""" + m, n = A.shape + if isinstance(A, Parameter): + return lambda p, c: make(p, c, "dense", None, m, n) + if sps.issparse(A.value): + V = A.value + return lambda c: make(None, c, "sparse", V.data, V.indices.astype(np.int32), + V.indptr.astype(np.int32), m, n) + data = np.ascontiguousarray(A.value).ravel() # row-major + return lambda c: make(None, c, "dense", data, m, n) + + +def index_flat(x, flat_indices, shape): + """Gather entries of ``x`` by column-major flat indices into ``shape``.""" + x = cast_to_const(x) + idx = np.asarray(flat_indices, dtype=np.int32).ravel() + shape = _check_shape(*shape) + if idx.size != shape[0] * shape[1]: + raise ValueError(f"{idx.size} indices do not fill shape {shape}") + if idx.size and (idx.min() < 0 or idx.max() >= x.size): + raise IndexError(f"flat index out of range for size {x.size}") + return Node(lambda c: _C.make_index(c, shape[0], shape[1], idx), [x], shape) + + +def _axis_indices(key, n): + """Positions along an axis of length n chosen by an int, slice or int list.""" + if isinstance(key, (bool, np.bool_)): # bool subclasses int; x[True] is not x[1] + raise IndexError("Boolean indices are not supported; use integer indices") + if isinstance(key, (int, np.integer)): + if not -n <= key < n: + raise IndexError(f"Index {key} out of range for axis of length {n}") + return np.array([key % n]) + if isinstance(key, slice): + return np.arange(n)[key] + if isinstance(key, (list, np.ndarray)): + key = np.asarray(key) + if key.ndim != 1 or not np.issubdtype(key.dtype, np.integer): + raise IndexError("Index arrays must be 1-D with integer entries " + f"(got shape {key.shape}, dtype {key.dtype})") + if key.size and (key.min() < -n or key.max() >= n): + raise IndexError(f"Index out of range for axis of length {n}") + return key % n + raise IndexError(f"Unsupported index type: {type(key).__name__}") + + +def _index(x, key): + """x[i, j] on anything; x[i] only on vectors, where it indexes along the axis.""" + d1, d2 = x.shape + if isinstance(key, tuple): + if len(key) != 2: + raise IndexError( + f"Expected 2 indices for shape {x.shape}, got {len(key)}") + rows, cols = _axis_indices(key[0], d1), _axis_indices(key[1], d2) + flat = (rows[:, None] + cols[None, :] * d1).ravel(order="F") + return index_flat(x, flat, (rows.size, cols.size)) + if d1 > 1 and d2 > 1: + raise IndexError(f"A single index on shape {x.shape} is ambiguous; " + "use X[i, j], X[i, :] or X[:, j]") + flat = _axis_indices(key, x.size) + return index_flat(x, flat, (1, flat.size) if d1 == 1 else (flat.size, 1)) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..bd8e38b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,42 @@ +import numpy as np +import pytest + +import sparsediffpy as sp + + +@pytest.fixture +def scope(): + return sp.Scope() + + +@pytest.fixture +def rng(): + return np.random.default_rng(0) + + +def check_derivatives(fn, scope, x0, w, h=1e-6, rtol=1e-5, atol=1e-6): + """Check fn.jacobian() and fn.hessian(w) against central differences.""" + x0 = np.asarray(x0, dtype=np.float64) + + def at(x, f): + scope.set_values(x) + return f() + + def jac(x): + return at(x, fn.jacobian).toarray() + + J = jac(x0) + w = np.asarray(w, dtype=np.float64).ravel(order="F") + W = w.reshape(fn.shape, order="F") # hessian() takes weights shaped like f + H = at(x0, lambda: fn.hessian(W)).toarray() + J_fd = np.zeros_like(J) + H_fd = np.zeros_like(H) + for j in range(x0.size): + e = np.zeros_like(x0) + e[j] = h + df = at(x0 + e, fn.forward) - at(x0 - e, fn.forward) + J_fd[:, j] = df.ravel(order="F") / (2 * h) + H_fd[:, j] = (jac(x0 + e) - jac(x0 - e)).T @ w / (2 * h) + scope.set_values(x0) + np.testing.assert_allclose(J, J_fd, rtol=rtol, atol=atol, err_msg="Jacobian") + np.testing.assert_allclose(H, H_fd, rtol=rtol, atol=atol, err_msg="Hessian") diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..9329d82 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,494 @@ +"""Scope/values, shape rules, validation errors, parameters, and Problem.""" + +import sys + +import numpy as np +import pytest + +import sparsediffpy as sp +from sparsediffpy import _compile + + +def test_values_are_column_major(scope): + X, y = scope.Variable(2, 2), scope.Variable(1, 1) + X.value = np.array([[1.0, 2.0], [3.0, 4.0]]) + y.value = 5.0 + np.testing.assert_array_equal(scope.get_values(), [1, 3, 2, 4, 5]) + scope.set_values(np.arange(5.0)) + np.testing.assert_array_equal(X.value, [[0, 2], [1, 3]]) + + +def test_wrong_sizes(scope): + x = scope.Variable(3, 1) + p = scope.Parameter(2, 1) + with pytest.raises(sp.DimensionError): + x.value = [1.0, 2.0] + with pytest.raises(sp.DimensionError): + p.value = [1.0, 2.0, 3.0] + with pytest.raises(sp.DimensionError): + scope.set_values(np.zeros(4)) + assert p.value is None + + +def test_value_shapes(scope): + """Values need the exact shape, not just the right number of elements.""" + X, r, c, y = (scope.Variable(2, 3), scope.Variable(1, 3), + scope.Variable(3, 1), scope.Variable(1, 1)) + P = scope.Parameter(2, 3) + for leaf in (X, P): + for bad in (np.zeros(6), np.zeros((3, 2)), np.zeros((6, 1)), 1.0): + with pytest.raises(sp.DimensionError, match="expected shape"): + leaf.value = bad + leaf.value = np.arange(6.0).reshape(2, 3) + np.testing.assert_array_equal(leaf.value, np.arange(6.0).reshape(2, 3)) + with pytest.raises(sp.DimensionError): + r.value = np.zeros((3, 1)) + with pytest.raises(sp.DimensionError): + c.value = np.zeros((1, 3)) + r.value, c.value, y.value = [1.0, 2.0, 3.0], [4.0, 5.0, 6.0], 7.0 + np.testing.assert_array_equal(r.value, [[1, 2, 3]]) + np.testing.assert_array_equal(c.value, [[4], [5], [6]]) + y.value = [8.0] + assert y.value.shape == (1, 1) + + n = scope.total_var_size + with pytest.raises(sp.DimensionError, match="values"): + scope.set_values(np.zeros((n, 1))) + scope.set_values(np.zeros(n)) + + fn = sp.compile(sp.exp(X)) + with pytest.raises(sp.DimensionError): + fn.hessian(np.ones(6)) + assert fn.hessian(np.ones((2, 3))).shape == (n, n) + fn = sp.compile(sp.exp(c)) + assert fn.hessian(np.ones(3)).shape == fn.hessian(np.ones((3, 1))).shape + with pytest.raises(sp.DimensionError): + fn.hessian(np.ones((1, 3))) + + +@pytest.mark.parametrize("shape", [(0, 1), (-1, 2), (2.0, 1)]) +def test_invalid_shapes(scope, shape): + with pytest.raises((ValueError, TypeError)): + scope.Variable(*shape) + + +def test_reevaluates_at_new_values(scope): + x = scope.Variable(2, 1) + fn = sp.compile(sp.exp(x)) + for v in ([0.0, 1.0], [2.0, -1.0]): + x.value = v + np.testing.assert_allclose(fn.forward(), np.exp(v)[:, None]) + np.testing.assert_allclose(fn.jacobian().diagonal(), np.exp(v)) + np.testing.assert_allclose(fn.hessian([1, 1]).diagonal(), np.exp(v)) + + +def test_two_compiles_share_scope(scope): + x, y = scope.Variable(2, 1), scope.Variable(2, 1) + f, g = sp.compile(sp.sin(x)), sp.compile(x * y) + scope.set_values([1.0, 2.0, 3.0, 4.0]) + np.testing.assert_allclose(f.forward(), np.sin([[1], [2]])) + np.testing.assert_allclose(g.forward(), [[3], [8]]) + assert f.jacobian().shape == g.jacobian().shape == (2, 4) + + +def test_forward_has_expression_shape(scope): + x, M = scope.Variable(1, 3), scope.Variable(2, 2) + scope.set_values([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]) + assert sp.compile(2.0 * x).forward().shape == (1, 3) + out = sp.compile(sp.exp(M)).forward() + assert out.shape == (2, 2) + np.testing.assert_allclose(out, np.exp([[4.0, 6.0], [5.0, 7.0]])) # column-major + + +def test_constants_must_be_2d(scope): + x = scope.Variable(3, 1) + for bad in (np.ones(3), [1.0, 2.0, 3.0], np.zeros((2, 2, 2))): + with pytest.raises(ValueError, match="2-D"): + x + bad + with pytest.raises(ValueError, match="2-D"): + bad * x + assert (x + np.array([2.0])).shape == (x + 2.0).shape == (3, 1) + assert (x + np.ones((3, 1))).shape == (3, 1) + + +def test_variable_declared_after_compile(scope): + x = scope.Variable(2, 1) + fn = sp.compile(sp.exp(x)) + scope.Variable(3, 1) + x.value = [0.0, 1.0] + np.testing.assert_allclose(fn.forward(), np.exp([[0.0], [1.0]])) + assert fn.jacobian().shape == (2, 2) + + +def test_affine_hessian_is_zero(scope): + x = scope.Variable(3, 1) + fn = sp.compile(sp.sum(-x[0:2]) + 2.0) + x.value = [1.0, 2.0, 3.0] + assert not fn.hessian([1.0]).toarray().any() + + +def test_shapes(scope): + x, r, X = scope.Variable(3, 1), scope.Variable(1, 2), scope.Variable(3, 2) + assert (x + r).shape == (3, 2) + assert (X @ r.T).shape == (3, 1) + assert x.T.shape == (1, 3) + assert x[1].shape == (1, 1) and x[-2:].shape == (2, 1) + assert r[1].shape == (1, 1) and X[:, 1].shape == (3, 1) + assert sp.sum(X, axis=1).shape == (1, 3) + + +@pytest.mark.parametrize("make", [ + lambda s: s.Variable(3, 1) + s.Variable(2, 1), + lambda s: s.Variable(3, 2) @ s.Variable(3, 2), + lambda s: sp.trace(s.Variable(2, 3)), + lambda s: sp.diag_vec(s.Variable(1, 3)), + lambda s: sp.reshape(s.Variable(3, 2), 4, 2), + lambda s: sp.hstack([]), + lambda s: sp.vstack([s.Variable(2, 1), s.Variable(2, 2)]), + lambda s: sp.quad_form(s.Variable(3, 1), np.eye(2)), + lambda s: sp.sum(s.Variable(2, 2), axis=2), + lambda s: sp.rel_entr(s.Variable(3, 1), s.Variable(2, 1)), +]) +def test_shape_errors(scope, make): + with pytest.raises(ValueError): + make(scope) + + +@pytest.mark.parametrize("make", [ + lambda x: sp.log(x[1:3]), + lambda x: sp.entr(2.0 * x), + lambda x: sp.prod(sp.exp(x)), + lambda x: sp.quad_over_lin(x, x[0]), + lambda x: sp.rel_entr(x, x), +]) +def test_variable_only_atoms(scope, make): + with pytest.raises(ValueError): + make(scope.Variable(4, 1)) + + +@pytest.mark.parametrize("key", [4, -5, [0, 4], (0, 3), (3, 0)]) +def test_index_out_of_range(scope, key): + X = scope.Variable(2, 2) if isinstance(key, tuple) else scope.Variable(4, 1) + with pytest.raises(IndexError): + X[key] + + +def test_index_must_be_explicit(scope): + x, X = scope.Variable(4, 1), scope.Variable(2, 3) + for key in (1, slice(0, 2), [0, 1]): + with pytest.raises(IndexError, match="ambiguous"): + X[key] + assert X[1, :].shape == (1, 3) and X[:, 1].shape == (2, 1) + with pytest.raises(IndexError, match="integer"): + x[[True, False, True, False]] + for key in (True, False, np.True_, (True, 0), (0, False)): + with pytest.raises(IndexError, match="Boolean"): + x[key] + with pytest.raises(IndexError, match="1-D"): + x[np.array([[0, 1]])] + with pytest.raises(IndexError, match="2 indices"): + X[0, 0, 0] + + +def test_compile_errors(scope): + with pytest.raises(ValueError, match="at least one Variable"): + sp.compile(sp.Scope().Parameter(1, 1, value=1.0) * 2.0) + with pytest.raises(ValueError, match="same Scope"): + sp.compile(scope.Variable(1, 1) + sp.Scope().Variable(1, 1)) + with pytest.raises(TypeError): + scope.Variable(1, 1) ** scope.Variable(1, 1) + + +def test_expressions_are_not_iterable(scope): + x, X = scope.Variable(3, 1), scope.Variable(2, 2) + with pytest.raises(TypeError, match="not iterable"): + list(x) + with pytest.raises(TypeError, match="not iterable"): + sp.Problem(sp.sum(X), X) # constraints must be a list + assert sp.Problem(sp.sum(X), [X]).total_constraint_size == 4 + + +def test_none_is_rejected(scope): + x, p = scope.Variable(2, 1), scope.Parameter(2, 1) + with pytest.raises(TypeError, match="None"): + x + None + with pytest.raises(TypeError, match="None"): + x.value = None + with pytest.raises(TypeError, match="None"): + p.value = None + + +def test_index_flat_wraps_constants(scope): + x = scope.Variable(2, 1) + fn = sp.compile(sp.index_flat(np.array([[5.0], [6.0]]), [1], (1, 1)) + x[0]) + x.value = [1.0, 2.0] + np.testing.assert_array_equal(fn.forward(), [[7.0]]) + + +def test_numpy_scalar_exponent(scope): + x = scope.Variable(2, 1) + f1, f2 = sp.compile(x ** np.int64(2)), sp.compile(x ** np.float32(2)) + x.value = [1.5, -2.0] + np.testing.assert_array_equal(f1.forward(), x.value ** 2) + np.testing.assert_array_equal(f2.forward(), x.value ** 2) + + +def test_division_only_by_nonzero_constants(scope): + x = scope.Variable(2, 1) + with pytest.raises(TypeError, match="d \\*\\* -1"): + x / x + with pytest.raises(TypeError): + 1.0 / x + with pytest.raises(ZeroDivisionError): + x / np.array([[1.0], [0.0]]) + + +def test_quad_form_parameter_must_be_symmetric(scope): + x = scope.Variable(2, 1) + N, S = np.array([[1.0, 5.0], [0.0, 1.0]]), np.array([[1.0, 2.0], [2.0, 3.0]]) + with pytest.raises(ValueError, match="symmetric"): + sp.quad_form(x, scope.Parameter(2, 2, value=N)) + P = scope.Parameter(2, 2, value=S) + fn = sp.compile(sp.quad_form(x, P)) + with pytest.raises(ValueError, match="symmetric"): + P.value = N + x.value = [1.0, 1.0] + P.value = 2 * S + np.testing.assert_allclose(fn.jacobian().toarray(), [2 * (2 * S) @ [1.0, 1.0]]) + + +def test_parameter_update_without_recompile(scope): + x = scope.Variable(3, 1) + A = scope.Parameter(2, 3, value=np.ones((2, 3))) + c = scope.Parameter(1, 1, value=1.0) + fn = sp.compile(c * sp.exp(A @ x)) + x.value = [0.1, 0.2, 0.3] + for Av, cv in [(np.ones((2, 3)), 1.0), (np.arange(6.0).reshape(2, 3), -2.0)]: + A.value, c.value = Av, cv + np.testing.assert_allclose(fn.forward(), cv * np.exp(Av @ x.value)) + J = cv * np.exp(Av @ x.value) * Av + np.testing.assert_allclose(fn.jacobian().toarray(), J) + + +def test_unset_parameter_raises(scope): + x, p = scope.Variable(2, 1), scope.Parameter(2, 1) + fn = sp.compile(p * x) + with pytest.raises(ValueError, match="no value"): + fn.forward() + p.value = [1.0, 2.0] + np.testing.assert_allclose(fn.forward(), [[0], [0]]) + + +def count_calls(monkeypatch, name): + """Count calls to the engine function `name` as seen from _compile.""" + calls, real = [], getattr(_compile._C, name) + + def counted(*args): + calls.append(1) + return real(*args) + monkeypatch.setattr(_compile._C, name, counted) + return calls + + +def test_forward_runs_once_per_point(scope, monkeypatch): + n_fwd = count_calls(monkeypatch, "problem_constraint_forward") + x, p = scope.Variable(2, 1), scope.Parameter(1, 1, value=1.0) + fn = sp.compile(p * sp.exp(x)) + x.value = [0.0, 1.0] + for _ in range(3): + np.testing.assert_allclose(fn.forward(), np.exp([[0.0], [1.0]])) + assert len(n_fwd) == 1 + x.value = [1.0, 1.0] + fn.forward() + scope.set_values([2.0, 2.0]) + fn.forward() + assert len(n_fwd) == 3 + p.value = 3.0 + np.testing.assert_allclose(fn.forward(), 3.0 * np.exp([[2.0], [2.0]])) + assert len(n_fwd) == 4 + + +def test_derivatives_reuse_forward_and_jacobian(scope, monkeypatch): + n_fwd = count_calls(monkeypatch, "problem_constraint_forward") + n_jac = count_calls(monkeypatch, "problem_jacobian") + n_hess = count_calls(monkeypatch, "problem_hessian") + x = scope.Variable(2, 1) + fn = sp.compile(sp.exp(x)) + x.value = [0.5, -0.5] + fn.forward() + J = fn.jacobian() + assert fn.jacobian() is J + fn.hessian([1.0, 1.0]) + fn.hessian([2.0, 0.0]) + assert (len(n_fwd), len(n_jac), len(n_hess)) == (1, 1, 2) + x.value = [0.0, 0.0] + fn.hessian([1.0, 1.0]) + assert (len(n_fwd), len(n_jac), len(n_hess)) == (2, 2, 3) + + +def test_problem_skips_repeated_u(scope, monkeypatch): + n_fwd = count_calls(monkeypatch, "problem_constraint_forward") + x = scope.Variable(2, 1) + prob = sp.Problem(sp.sum(x), [sp.exp(x)]) + prob.constraint_forward(np.array([1.0, 2.0])) + prob.constraint_forward(np.array([1.0, 2.0])) # equal, distinct array + assert len(n_fwd) == 1 + prob.constraint_forward(np.array([1.0, 3.0])) + assert len(n_fwd) == 2 + prob.constraint_forward() # scope values are a different point + assert len(n_fwd) == 3 + np.testing.assert_allclose(prob.constraint_forward(), [1.0, 1.0]) + assert len(n_fwd) == 3 + + +def test_derivatives_require_forward(scope): + x = scope.Variable(2, 1) + prob = sp.Problem(sp.sum(x), [sp.exp(x)]) + with pytest.raises(RuntimeError, match="objective_forward"): + prob.gradient() + with pytest.raises(RuntimeError, match="constraint_forward"): + prob.jacobian() + prob.objective_forward([1.0, 2.0]) + prob.constraint_forward([1.0, 3.0]) + with pytest.raises(RuntimeError, match="same point"): + prob.hessian(1.0, [1.0, 1.0]) + prob.objective_forward([1.0, 2.0]) # a point is "the last u": redo both forwards + prob.constraint_forward([1.0, 2.0]) + assert prob.hessian(1.0, [1.0, 1.0]).shape == (2, 2) + + +def test_problem(scope): + x = scope.Variable(3, 1) + shared = sp.sin(x) + prob = sp.Problem(sp.sum(sp.exp(shared)), [x[0:2] * x[1:3], shared]) + u = np.array([0.3, -0.5, 0.8]) + s, c = np.sin(u), np.cos(u) + + assert prob.objective_forward(u) == pytest.approx(np.exp(s).sum()) + np.testing.assert_allclose(prob.gradient(), np.exp(s) * c) + np.testing.assert_allclose(prob.constraint_forward(u), + np.r_[u[:2] * u[1:], s]) + + J = prob.jacobian().toarray() + np.testing.assert_allclose(J[:2], [[u[1], u[0], 0], [0, u[2], u[1]]]) + np.testing.assert_allclose(J[2:], np.diag(c)) + + lam = np.array([1.0, 2.0, 0.5, -1.0, 3.0]) + H = prob.hessian(2.0, lam).toarray() + H_obj = np.diag(np.exp(s) * (c ** 2 - s)) + H_con = np.array([[0, 1, 0], [1, 0, 2], [0, 2, 0]]) + np.diag(-s * lam[2:]) + np.testing.assert_allclose(H, 2.0 * H_obj + H_con) + + with pytest.raises(ValueError, match="lagrange"): + prob.hessian(1.0, np.ones(4)) + for bad in (np.zeros(2), np.zeros(4)): + with pytest.raises(sp.DimensionError, match="1-D array of 3 elements"): + prob.objective_forward(bad) + with pytest.raises(sp.DimensionError, match="1-D array of 3 elements"): + prob.constraint_forward(bad) + with pytest.raises(sp.DimensionError): + prob.objective_forward(u[:, None]) + with pytest.raises(sp.DimensionError, match="lagrange"): + prob.hessian(1.0, lam[:, None]) + with pytest.raises(ValueError, match="scalar"): + sp.Problem(x) + + +def test_problem_parameters(scope): + x, a = scope.Variable(2, 1), scope.Parameter(1, 1, value=2.0) + prob = sp.Problem(sp.sum(a * x ** 2)) + u = np.array([1.0, 3.0]) + assert prob.objective_forward(u) == pytest.approx(20.0) + a.value = 0.5 + assert prob.objective_forward(u) == pytest.approx(5.0) + np.testing.assert_allclose(prob.gradient(), u) + + +def test_objective_and_constraints_at_different_points(scope): + x, a = scope.Variable(3, 1), scope.Parameter(1, 1, value=2.0) + e = sp.exp(a * x) # shared by the objective and the constraint + prob = sp.Problem(sp.sum(e), [e]) + prob.objective_forward(np.zeros(3)) + prob.constraint_forward(np.ones(3)) + np.testing.assert_allclose(prob.gradient(), [2.0, 2.0, 2.0]) + np.testing.assert_allclose(prob.jacobian().diagonal(), 2 * np.exp([2.0] * 3)) + + +def test_hessian_without_prior_jacobian(scope): + """hessian() must refresh the Jacobian values the engine's Hessian reuses.""" + x, y = scope.Variable(2, 1), scope.Variable(2, 1) + fn = sp.compile(x * y + sp.sin(x * y)) + prob = sp.Problem(sp.sum(x * y), [x * y]) + for u in ([0.1, 0.2, 0.3, 0.4], [1.0, -2.0, 0.5, 3.0]): + scope.set_values(u) + xy, xv, yv = np.multiply(u[:2], u[2:]), np.array(u[:2]), np.array(u[2:]) + w = np.array([1.0, 2.0]) + # d2/dx_i dy_i of w_i * (x_i y_i + sin(x_i y_i)) + d2 = lambda a, b: w * (-np.sin(xy) * a * b) # noqa: E731 + off = np.diag(w * (1 + np.cos(xy)) + d2(xv, yv)) + H = np.block([[np.diag(d2(yv, yv)), off], [off, np.diag(d2(xv, xv))]]) + np.testing.assert_allclose(fn.hessian(w).toarray(), H, atol=1e-12) + + prob.objective_forward(u) + prob.constraint_forward(u) + H_prob = np.block([[np.zeros((2, 2)), np.diag(2.0 + w)], + [np.diag(2.0 + w), np.zeros((2, 2))]]) + np.testing.assert_allclose(prob.hessian(2.0, w).toarray(), H_prob, + atol=1e-12) + + +def test_compile_deep_sum(scope): + """A loop-built sum is a chain of adds deeper than the recursion limit.""" + x, p = scope.Variable(3, 1), scope.Parameter(1, 1, value=2.0) + n_adds = 5000 + assert n_adds > sys.getrecursionlimit() # each add is one level deeper + f = x[0] + for i in range(1, n_adds + 1): + f = f + p * sp.sin(x[i % 3]) + fn = sp.compile(f) + u = np.array([0.3, -0.7, 1.1]) + x.value = u + counts = np.bincount(np.arange(1, n_adds + 1) % 3, minlength=3) + for pv in (2.0, 3.0): + p.value = pv + np.testing.assert_allclose(fn.forward(), [[u[0] + pv * counts @ np.sin(u)]]) + np.testing.assert_allclose(fn.jacobian().toarray(), + [np.eye(3)[0] + pv * counts * np.cos(u)]) + np.testing.assert_allclose(fn.hessian(1.0).toarray(), + np.diag(-pv * counts * np.sin(u)), atol=1e-9) + + +def test_compile_deep_nonlinear_chain(scope): + """A deep chain of nonlinear atoms that all reuse one shared subexpression. + + The shared node is shallow on purpose: the engine re-evaluates a shared + subexpression once per use, so reusing a deep one would take exponential time. + """ + x = scope.Variable(1, 1) + n_levels, c = 3000, 1e-3 + shared = sp.sin(x) + g = x + for _ in range(n_levels): + g = sp.sin(g) + c * shared + + x0 = 0.7 + v, d1, d2 = x0, 1.0, 0.0 # g, dg/dx, d2g/dx2 by the chain rule + for _ in range(n_levels): + d2 = np.cos(v) * d2 - np.sin(v) * d1 ** 2 - c * np.sin(x0) + d1 = np.cos(v) * d1 + c * np.cos(x0) + v = np.sin(v) + c * np.sin(x0) + + x.value = x0 + fn = sp.compile(g) + np.testing.assert_allclose(fn.forward(), [[v]], rtol=1e-12) + np.testing.assert_allclose(fn.jacobian().toarray(), [[d1]], rtol=1e-10) + np.testing.assert_allclose(fn.hessian(1.0).toarray(), [[d2]], rtol=1e-8) + + prob = sp.Problem(sp.sum(g), [g]) + u = np.array([x0]) + assert prob.objective_forward(u) == pytest.approx(v, rel=1e-12) + np.testing.assert_allclose(prob.gradient(), [d1], rtol=1e-10) + np.testing.assert_allclose(prob.constraint_forward(u), [v], rtol=1e-12) + np.testing.assert_allclose(prob.jacobian().toarray(), [[d1]], rtol=1e-10) + np.testing.assert_allclose(prob.hessian(1.0, [2.0]).toarray(), [[3 * d2]], + rtol=1e-8) diff --git a/tests/test_atoms.py b/tests/test_atoms.py new file mode 100644 index 0000000..48d4e8b --- /dev/null +++ b/tests/test_atoms.py @@ -0,0 +1,121 @@ +"""Each atom is wired to the right engine call. + +A case is ``(name, shapes, f, positive)``. ``f(m, *xs)`` is evaluated once +with ``m = sp`` on Variables (the expression under test) and once with +``m = NP`` on their values (the expected result). A shape that is an array +declares a Parameter with that value. The value must match NumPy, and the +Jacobian and Hessian must match finite differences. +""" + +from types import SimpleNamespace + +import numpy as np +import pytest +import scipy.sparse as sps +from conftest import check_derivatives +from scipy.special import ndtr + +import sparsediffpy as sp + +A = np.array([[1.0, -2.0, 0.5], [0.0, 3.0, 1.0]]) +Q = np.array([[2.0, 1.0, 0.0], [1.0, 3.0, 0.5], [0.0, 0.5, 4.0]]) +N = Q + np.triu(np.ones((3, 3)), 1) # not symmetric + + +def _reduce(f): + """NumPy sum/prod with sp's shapes: (1, 1), or a row for axis 0 or 1.""" + return lambda X, axis=None: np.reshape(f(X, axis=axis), (1, -1)) + + +def _dense(M): + return M.toarray() if sps.issparse(M) else M + + +NP = SimpleNamespace( + sin=np.sin, cos=np.cos, exp=np.exp, sinh=np.sinh, tanh=np.tanh, log=np.log, + tan=np.tan, asinh=np.arcsinh, atan=np.arctan, atanh=np.arctanh, + logistic=lambda x: np.logaddexp(0, x), normal_cdf=ndtr, + xexp=lambda x: x * np.exp(x), entr=lambda x: -x * np.log(x), + power=lambda x, p: x ** p, atan2=np.arctan2, + sum=_reduce(np.sum), prod=_reduce(np.prod), hstack=np.hstack, vstack=np.vstack, + reshape=lambda x, d1, d2: x.reshape(d1, d2, order="F"), + trace=lambda X: np.trace(X).reshape(1, 1), diag_vec=lambda x: np.diag(x.ravel()), + quad_form=lambda x, P: x.T @ _dense(P) @ x, + quad_over_lin=lambda x, z: (x ** 2).sum() / z, + rel_entr=lambda x, y: x * np.log(x / y), +) + +ELEMENTWISE = ["sin", "cos", "exp", "sinh", "tanh", "asinh", "atan", "logistic", + "normal_cdf", "xexp"] +POSITIVE_ELEMENTWISE = ["log", "tan", "atanh", "entr"] + +CASES = [ + *[(a, [(3, 2)], lambda m, x, a=a: getattr(m, a)(x), False) for a in ELEMENTWISE], + *[(a, [(3, 2)], lambda m, x, a=a: getattr(m, a)(x), True) + for a in POSITIVE_ELEMENTWISE], + ("power", [(3, 2)], lambda m, x: x ** 3 + m.power(x, 0.5), True), + ("atan2", [(3, 1), (3, 1)], lambda m, y, x: m.atan2(y, x), True), + # affine + ("add_broadcast", [(1, 1), (1, 2), (3, 1), (3, 2)], + lambda m, a, r, c, X: a * X + r + c - 1.0, False), + ("neg_sub", [(3, 1), (3, 1)], lambda m, x, y: -x - y + np.c_[0:3.0], False), + ("constant_mult", [(3, 2)], lambda m, X: 2.0 * X * np.arange(6.0).reshape(3, 2), + False), + ("constant_div", [(3, 2), (1, 1)], + lambda m, X, a: X / 2.0 + a / np.arange(1.0, 7.0).reshape(3, 2), False), + ("matmul_dense", [(3, 1), (2, 3)], lambda m, x, Y: A @ x + (Y @ A.T)[:, 0:1], + False), + ("matmul_sparse", [(3, 1), (2, 3)], + lambda m, x, Y: sps.csr_matrix(A) @ x + (Y @ sps.csr_matrix(A.T))[:, 0:1], + False), + ("transpose_reshape", [(3, 2)], lambda m, X: m.reshape(X.T, 6, 1), False), + ("sums", [(3, 2)], + lambda m, X: m.hstack([m.sum(X), m.sum(X, axis=0), m.sum(X, axis=1)]), False), + ("trace_diag_vec", [(3, 3), (3, 1)], lambda m, X, x: m.trace(X) + m.diag_vec(x), + False), + ("stacking", [(2, 1), (2, 2)], + lambda m, x, y: m.vstack([m.hstack([x, y]), m.hstack([y, x])]), False), + ("indexing", [(3, 3), (1, 4)], + lambda m, X, r: m.vstack([X[0:2, [2, 0]], X[2:3, 1:], r[:, 1:3]] + if m is NP else [X[0:2, [2, 0]], X[-1, 1:], r[1:3]]), + False), + # bivariate and other + ("multiply", [(3, 1), (3, 2)], lambda m, x, Y: m.sin(x) * Y, False), + ("matmul_variables", [(2, 3), (3, 2)], lambda m, X, Y: X @ Y, False), + ("x_transpose_x", [(3, 1)], lambda m, x: x.T @ x, False), + ("quad_form", [(3, 1), 2 * Q], + lambda m, x, P: m.vstack([m.quad_form(x, Q), m.quad_form(x, sps.csr_matrix(Q)), + m.quad_form(x, P)]), False), + ("quad_form_nonsymmetric", [(3, 1)], + lambda m, x: m.quad_form(x, N) + m.quad_form(x, sps.csr_matrix(N)), False), + ("quad_over_lin", [(3, 1), (1, 1)], lambda m, x, z: m.quad_over_lin(x, z), True), + ("rel_entr", [(3, 1), (3, 1)], lambda m, x, y: m.rel_entr(x, y), True), + ("rel_entr_scalar_vector", [(1, 1), (3, 1)], lambda m, x, y: m.rel_entr(x, y), + True), + ("rel_entr_vector_scalar", [(3, 1), (1, 1)], lambda m, x, y: m.rel_entr(x, y), + True), + ("prods", [(3, 2)], + lambda m, X: m.hstack([m.prod(X), m.prod(X, axis=0), m.prod(X, axis=1)]), True), + # compositions + ("parameter_matmul_and_scale", [(3, 1), A, np.array([[3.0]])], + lambda m, x, B, c: c * m.exp(B @ x) + B @ m.sin(x), False), + ("shared_subexpression", [(3, 1)], + lambda m, x: (lambda u: u * u + m.exp(u))(m.sin(A @ x)), False), +] + + +@pytest.mark.parametrize("name, shapes, f, positive", CASES, + ids=[c[0] for c in CASES]) +def test_atom(name, shapes, f, positive, rng): + scope = sp.Scope() + leaves = [scope.Parameter(*s.shape, value=s) if isinstance(s, np.ndarray) + else scope.Variable(*s) for s in shapes] + expr = f(sp, *leaves) + fn = sp.compile(expr) + low, high = (0.2, 0.9) if positive else (-1.0, 1.0) + x0 = rng.uniform(low, high, scope.total_var_size) + scope.set_values(x0) + expected = f(NP, *[leaf.value for leaf in leaves]) + np.testing.assert_allclose(fn.forward(), np.broadcast_to(expected, expr.shape), + rtol=1e-10, atol=1e-12) + check_derivatives(fn, scope, x0, rng.standard_normal(expr.size))