From afaab489939f27a29c2b0389447190f7cd43c7c6 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 14:03:50 -0800 Subject: [PATCH 01/16] Add sp.compact: merge scalar loop terms into vector expressions sp.compact(terms) is a drop-in for sp.vstack(terms). Terms with the same template (same operations, differing only in variable entries and scalar constants) are rebuilt once as a vector expression; other terms are kept. It returns (expr, perm), where row j of expr is terms[perm[j]], or perm is None when the order is unchanged. Nodes now carry an op name, and index/power carry the attrs the template reads. Co-Authored-By: Claude Opus 5.5 --- README.md | 26 ++++++++ sparsediffpy/__init__.py | 1 + sparsediffpy/_atoms.py | 28 ++++---- sparsediffpy/_compact.py | 125 ++++++++++++++++++++++++++++++++++++ sparsediffpy/_expression.py | 40 ++++++++---- tests/test_compact.py | 83 ++++++++++++++++++++++++ 6 files changed, 275 insertions(+), 28 deletions(-) create mode 100644 sparsediffpy/_compact.py create mode 100644 tests/test_compact.py diff --git a/README.md b/README.md index b11e3b7..39687d4 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,32 @@ res = minimize(prob.objective_forward, np.zeros(4), jac=grad, hess=obj_hess, print(res.x) # approx. [0. 0.289 0.549 0.784], on the boundary ||x|| = 1 ``` +## Models written as loops + +A model built term by term in a Python loop gives the engine one small scalar +expression per term, which is much slower than the same model written with +vectors. `sp.compact(terms)` is a drop-in for `sp.vstack(terms)` that merges +terms with the same structure (differing only in which entries and constants +they use) into one vector expression: + +```python +import numpy as np + +import sparsediffpy as sp + +n = 1000 +scope = sp.Scope() +x = scope.Variable(n, 1) +f, _ = sp.compact([(x[k] - 1.0) ** 2 for k in range(n)]) +g, perm = sp.compact([x[k] ** 2 + x[k + 1] - 1.0 for k in range(n - 1)]) +prob = sp.Problem(sp.sum(f), [g]) # 2 vector expressions, not ~2000 scalar ones +assert perm is None # one template per list, so rows kept their order +``` + +When a list mixes templates, rows are grouped by template: row `j` of the result +is `terms[perm[j]]`. Map row data such as bounds in with `values[perm]`, and +multipliers back out with `out[perm] = lam`. + ## Supported operations - **Operators:** `+ - * @ **`, `.T`, and indexing with ints, slices and integer lists: `x[1:3]` on a diff --git a/sparsediffpy/__init__.py b/sparsediffpy/__init__.py index 08995f9..ab3e2c8 100644 --- a/sparsediffpy/__init__.py +++ b/sparsediffpy/__init__.py @@ -24,3 +24,4 @@ sinh, sum, tan, tanh, trace, vstack, xexp, ) from sparsediffpy._compile import CompiledExpression, Problem, compile # noqa: F401 +from sparsediffpy._compact import compact # noqa: F401 diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/_atoms.py index 5bf22a4..b732f43 100644 --- a/sparsediffpy/_atoms.py +++ b/sparsediffpy/_atoms.py @@ -24,7 +24,7 @@ def atom(x): x = cast_to_const(x) if restricted: _require_variables(name, x) - return Node(make, [x], x.shape) + return Node(make, [x], x.shape, name) atom.__name__ = atom.__qualname__ = name atom.__doc__ = f"Elementwise {name}." return atom @@ -56,7 +56,7 @@ def atan2(y, x): _require_variables("atan2", y, x) if y.shape != x.shape: raise ValueError(f"atan2: shape mismatch {y.shape} vs {x.shape}") - return Node(_C.make_atan2, [y, x], y.shape) + return Node(_C.make_atan2, [y, x], y.shape, "atan2") def _reduction_shape(x, axis): @@ -73,7 +73,7 @@ def sum(x, axis=None): x = cast_to_const(x) shape = _reduction_shape(x, axis) c_axis = -1 if axis is None else axis - return Node(lambda c: _C.make_sum(c, c_axis), [x], shape) + return Node(lambda c: _C.make_sum(c, c_axis), [x], shape, "sum") def prod(x, axis=None): @@ -84,7 +84,7 @@ def prod(x, axis=None): shape = _reduction_shape(x, axis) make = {None: _C.make_prod, 0: _C.make_prod_axis_zero, 1: _C.make_prod_axis_one}[axis] - return Node(make, [x], shape) + return Node(make, [x], shape, "prod") def reshape(x, d1, d2): @@ -92,14 +92,14 @@ def reshape(x, d1, d2): x, shape = cast_to_const(x), _check_shape(d1, d2) if x.size != shape[0] * shape[1]: raise ValueError(f"Cannot reshape {x.shape} to {shape}") - return Node(lambda c: _C.make_reshape(c, *shape), [x], shape) + return Node(lambda c: _C.make_reshape(c, *shape), [x], shape, "reshape") def trace(x): x = cast_to_const(x) if x.shape[0] != x.shape[1]: raise ValueError(f"trace requires a square matrix, got {x.shape}") - return Node(_C.make_trace, [x], (1, 1)) + return Node(_C.make_trace, [x], (1, 1), "trace") def diag_vec(x): @@ -107,7 +107,7 @@ def diag_vec(x): x = cast_to_const(x) if x.shape[1] != 1: raise ValueError(f"diag_vec requires a column vector, got {x.shape}") - return Node(_C.make_diag_vec, [x], (x.shape[0], x.shape[0])) + return Node(_C.make_diag_vec, [x], (x.shape[0], x.shape[0]), "diag_vec") def hstack(exprs): @@ -117,7 +117,7 @@ def hstack(exprs): raise ValueError(f"hstack: need >=1 args with equal rows, got " f"{[e.shape for e in exprs]}") shape = (exprs[0].shape[0], np.sum([e.shape[1] for e in exprs]).item()) - return Node(lambda *caps: _C.make_hstack(list(caps)), exprs, shape) + return Node(lambda *caps: _C.make_hstack(list(caps)), exprs, shape, "hstack") def vstack(exprs): @@ -127,7 +127,7 @@ def vstack(exprs): raise ValueError(f"vstack: need >=1 args with equal columns, got " f"{[e.shape for e in exprs]}") shape = (np.sum([e.shape[0] for e in exprs]).item(), exprs[0].shape[1]) - return Node(lambda *caps: _C.make_vstack(list(caps)), exprs, shape) + return Node(lambda *caps: _C.make_vstack(list(caps)), exprs, shape, "vstack") def quad_form(x, Q): @@ -147,18 +147,18 @@ def quad_form(x, Q): if Q._flat is not None: _check_symmetric(Q._flat, Q.shape) return Node(lambda p, c: _C.make_quad_form(p, c, "dense", None, n), - [Q, x], (1, 1)) + [Q, x], (1, 1), "quad_form") if not isinstance(Q, Constant): raise TypeError("quad_form: Q must be a constant or a Parameter") V = (Q.value + Q.value.T) / 2 # the engine assumes symmetric Q if not sps.issparse(V): data = np.ascontiguousarray(V).ravel() return Node(lambda c: _C.make_quad_form(None, c, "dense", data, n), - [x], (1, 1)) + [x], (1, 1), "quad_form") V = sps.csr_matrix(V) return Node(lambda c: _C.make_quad_form( None, c, "sparse", V.data, V.indices.astype(np.int32), - V.indptr.astype(np.int32), n, n), [x], (1, 1)) + V.indptr.astype(np.int32), n, n), [x], (1, 1), "quad_form") def quad_over_lin(x, z): @@ -166,7 +166,7 @@ def quad_over_lin(x, z): _require_variables("quad_over_lin", x, z) if z.shape != (1, 1): raise ValueError(f"quad_over_lin: z must be scalar, got {z.shape}") - return Node(_C.make_quad_over_lin, [x, z], (1, 1)) + return Node(_C.make_quad_over_lin, [x, z], (1, 1), "quad_over_lin") def rel_entr(x, y): @@ -181,4 +181,4 @@ def rel_entr(x, y): shape = y.shape else: raise ValueError(f"rel_entr: incompatible shapes {x.shape} and {y.shape}") - return Node(_C.make_rel_entr, [x, y], shape) + return Node(_C.make_rel_entr, [x, y], shape, "rel_entr") diff --git a/sparsediffpy/_compact.py b/sparsediffpy/_compact.py new file mode 100644 index 0000000..1235b14 --- /dev/null +++ b/sparsediffpy/_compact.py @@ -0,0 +1,125 @@ +"""Merge scalar terms written in a Python loop into vector expressions. + +A loop such as ``[x[k] ** 2 + x[k + 1] - 1 for k in range(n - 1)]`` gives the +engine one small expression per term. The terms share a *template*: the same +operations, differing only in which entries of a variable and which scalar +constants they use. :func:`compact` rebuilds each template once as a vector +expression, with the varying parts gathered into index arrays and constants. +""" + +import numpy as np + +from sparsediffpy import _atoms +from sparsediffpy._expression import ( + Constant, Node, Parameter, Variable, cast_to_const, broadcast, index_flat, +) + +# Unary atoms that accept any argument (restricted ones need plain Variables). +_UNARY = {"sin", "cos", "exp", "sinh", "tanh", "asinh", "atan", "logistic", + "normal_cdf", "xexp"} + + +class _Unsupported(Exception): + pass + + +def compact(terms): + """Stack scalar expressions like :func:`vstack`, merging shared templates. + + Returns ``(expr, perm)``: ``expr`` has shape ``(len(terms), 1)`` and its + row ``j`` is ``terms[perm[j]]``, or ``perm`` is ``None`` if no row moved. + Terms using other operations (matmul, sum, ...) are kept as they are. + """ + if not terms: + raise ValueError("compact: need at least one term") + leaves, groups = {}, {} # leaves: id -> Variable/Parameter used by templates + for i, t in enumerate(terms): + t = cast_to_const(t) + if t.shape != (1, 1): + raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") + try: + key, holes = _template(t, leaves) + except _Unsupported: + key, holes = ("single", i), None + groups.setdefault(key, []).append((i, t, holes)) + + parts, perm = [], [] + for key, members in groups.items(): + perm += [i for i, _, _ in members] + parts.append(members[0][1] if len(members) == 1 else + _rebuild(key, [holes for _, _, holes in members], leaves)) + expr = parts[0] if len(parts) == 1 else _atoms.vstack(parts) + perm = None if perm == list(range(len(perm))) else np.array(perm) + return expr, perm + + +def _template(term, leaves): + """Describe a scalar term as ``(template, holes)``. + + The template lists the term's nodes in post-order. Each entry names the + operation and refers to its children by their position in the list, so a + subexpression shared inside the term is part of the template. Holes are + what varies between terms -- the entry index of ``x[i]`` and the value of a + scalar constant -- in the order they are met. + """ + entries, holes, seen = [], [], {} + + def visit(e): + if id(e) in seen: + return seen[id(e)] + if e.shape != (1, 1): + raise _Unsupported + if isinstance(e, Constant): + ent = ("const",) + holes.append(float(e.value[0, 0])) + elif isinstance(e, (Variable, Parameter)): + leaves[id(e)] = e + ent = ("leaf", id(e)) + elif not isinstance(e, Node): + raise _Unsupported + elif e.op == "index" and isinstance(e.args[0], Variable): + leaves[id(e.args[0])] = e.args[0] + ent = ("var", id(e.args[0])) + holes.append(int(e.attrs["idx"][0])) + elif e.op in ("add", "multiply", "scalar_mult"): + ent = (e.op, visit(e.args[0]), visit(e.args[1])) + elif e.op == "neg" or e.op in _UNARY: + ent = (e.op, visit(e.args[0])) + elif e.op == "power": + ent = ("power", e.attrs["p"], visit(e.args[0])) + else: + raise _Unsupported + seen[id(e)] = len(entries) + entries.append(ent) + return seen[id(e)] + + visit(term) + return tuple(entries), holes + + +def _rebuild(template, holes, leaves): + """Evaluate the template once on (K, 1) vectors, one row per term.""" + K = len(holes) + columns = iter(zip(*holes)) # the K values of each hole, in template order + vals = [] + for ent in template: + op = ent[0] + if op == "var": + vals.append(index_flat(leaves[ent[1]], next(columns), (K, 1))) + elif op == "const": + c = np.array(next(columns)) + vals.append(Constant(c[0] if np.all(c == c[0]) else c.reshape(K, 1))) + elif op == "leaf": + vals.append(leaves[ent[1]]) + elif op == "add": + vals.append(vals[ent[1]] + vals[ent[2]]) + elif op in ("multiply", "scalar_mult"): + vals.append(vals[ent[1]] * vals[ent[2]]) + elif op == "neg": + vals.append(-vals[ent[1]]) + elif op == "power": + vals.append(vals[ent[2]] ** ent[1]) + else: + vals.append(getattr(_atoms, op)(vals[ent[1]])) + # a template without holes (every term is e.g. the scalar y) builds a scalar + return broadcast(vals[-1], (K, 1)) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index 878bfdf..86e2b24 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -38,7 +38,7 @@ def size(self): @property def T(self): - return Node(_C.make_transpose, [self], self.shape[::-1]) + return Node(_C.make_transpose, [self], self.shape[::-1], "transpose") def __add__(self, other): return _add(self, cast_to_const(other)) @@ -53,7 +53,7 @@ def __rsub__(self, other): return _add(cast_to_const(other), -self) def __neg__(self): - return Node(_C.make_neg, [self], self.shape) + return Node(_C.make_neg, [self], self.shape, "neg") def __mul__(self, other): return _mul(self, cast_to_const(other)) @@ -70,7 +70,8 @@ def __rmatmul__(self, other): def __pow__(self, p): if not isinstance(p, numbers.Real): raise TypeError("Exponent must be a constant number") - return Node(lambda c: _C.make_power(c, float(p)), [self], self.shape) + return Node(lambda c: _C.make_power(c, float(p)), [self], self.shape, + "power", {"p": float(p)}) def __getitem__(self, key): return _index(self, key) @@ -82,12 +83,19 @@ def __iter__(self): class Node(Expression): - """An operation: ``make(*child_capsules)`` builds its C capsule.""" + """An operation: ``make(*child_capsules)`` builds its C capsule. - def __init__(self, make, args, shape): + ``op`` names the operation and ``attrs`` holds the parameters that rewrites + such as :func:`compact` read (``idx`` for index, ``p`` for power). + Compilation only uses ``make``. + """ + + def __init__(self, make, args, shape, op=None, attrs=None): self.make = make self.args = tuple(args) self.shape = tuple(shape) + self.op = op + self.attrs = attrs class Constant(Expression): @@ -254,23 +262,26 @@ def broadcast(x, shape): raise ValueError(f"Cannot broadcast {x.shape} to {shape}") if isinstance(x, Constant): return Constant(np.broadcast_to(x.dense(), shape)) - return Node(lambda c: _C.make_broadcast(c, *shape), [x], shape) + return Node(lambda c: _C.make_broadcast(c, *shape), [x], shape, + "broadcast") def _add(a, b): shape = _broadcast_shape(a.shape, b.shape) - return Node(_C.make_add, [broadcast(a, shape), broadcast(b, shape)], shape) + return Node(_C.make_add, [broadcast(a, shape), broadcast(b, shape)], shape, + "add") def _mul(a, b): if _is_data(b) and not _is_data(a): a, b = b, a # put the constant/parameter operand (if any) first if _is_data(a) and a.shape == (1, 1): - return Node(_C.make_param_scalar_mult, [a, b], b.shape) + return Node(_C.make_param_scalar_mult, [a, b], b.shape, "scalar_mult") shape = _broadcast_shape(a.shape, b.shape) a, b = broadcast(a, shape), broadcast(b, shape) - make = _C.make_param_vector_mult if _is_data(a) else _C.make_multiply - return Node(make, [a, b], shape) + if _is_data(a): + return Node(_C.make_param_vector_mult, [a, b], shape, "vector_mult") + return Node(_C.make_multiply, [a, b], shape, "multiply") def _matmul(a, b): @@ -279,11 +290,11 @@ def _matmul(a, b): shape = (a.shape[0], b.shape[1]) if _is_data(a): return Node(_matmul_with_data(_C.make_left_matmul, a), _data_args(a, b), - shape) + shape, "left_matmul") if _is_data(b): return Node(_matmul_with_data(_C.make_right_matmul, b), _data_args(b, a), - shape) - return Node(_C.make_matmul, [a, b], shape) + shape, "right_matmul") + return Node(_C.make_matmul, [a, b], shape, "matmul") def _data_args(data, x): @@ -312,7 +323,8 @@ def index_flat(x, flat_indices, shape): raise ValueError(f"{idx.size} indices do not fill shape {shape}") if idx.size and (idx.min() < 0 or idx.max() >= x.size): raise IndexError(f"flat index out of range for size {x.size}") - return Node(lambda c: _C.make_index(c, shape[0], shape[1], idx), [x], shape) + return Node(lambda c: _C.make_index(c, shape[0], shape[1], idx), [x], shape, + "index", {"idx": idx}) def _axis_indices(key, n): diff --git a/tests/test_compact.py b/tests/test_compact.py new file mode 100644 index 0000000..44f5faa --- /dev/null +++ b/tests/test_compact.py @@ -0,0 +1,83 @@ +"""compact(terms) must match vstack(terms) in value and derivatives.""" + +import numpy as np +import scipy.sparse as sps + +import sparsediffpy as sp +from sparsediffpy._expression import Constant + + +def _evaluate(g, u, lam): + # sum(g) is unchanged by a row permutation, so the objective needs no mapping + prob = sp.Problem(sp.sum(g), [g]) + f, c = prob.objective_forward(u), prob.constraint_forward(u) + return (f, prob.gradient().copy(), c.copy(), prob.jacobian().toarray(), + prob.hessian(0.7, lam).toarray()) + + +def assert_matches_vstack(build, n_vars, rng): + """build(scope) -> list of scalar terms; compare compact with vstack.""" + terms = build(sp.Scope()) + u = rng.uniform(0.5, 1.5, n_vars) + lam = rng.standard_normal(len(terms)) + f, grad, c, J, H = _evaluate(sp.vstack(terms), u, lam) + + g, perm = sp.compact(build(sp.Scope())) + rows = slice(None) if perm is None else perm + got = _evaluate(g, u, lam[rows]) + expected = (f, grad, c[rows], J[rows], H) + for name, a, b in zip(["objective", "gradient", "constraints", "jacobian", + "hessian"], expected, got): + np.testing.assert_allclose(b, a, rtol=1e-12, atol=1e-12, err_msg=name) + return g, perm + + +def test_loop_merges_into_one_vector(rng): + def build(scope): + x = scope.Variable(8, 1) + return [3 * x[k + 1] ** 3 + 2 * x[k + 2] - x[k] * sp.exp(x[k] - x[k + 1]) + for k in range(6)] + g, perm = assert_matches_vstack(build, 8, rng) + assert perm is None and g.op != "vstack" # a single template, in place + + +def test_varying_constants(rng): + def build(scope): + x = scope.Variable(5, 1) + return [(k + 1.0) * (x[k] - k) ** 2 for k in range(5)] + assert_matches_vstack(build, 5, rng) + + +def test_interleaved_templates_and_unsupported_terms(rng): + A = rng.standard_normal((2, 6)) + + def build(scope): + x = scope.Variable(6, 1) + terms = [] + for k in range(5): + terms += [sp.sin(x[k]) * x[k + 1], x[k] ** 2 - 2.0 * k] + # row @ x is not merged: kept as is, but still placed via perm + terms += [sp.exp(Constant(A[:1]) @ x), + sp.exp(Constant(sps.csr_matrix(A[1:])) @ x)] + return terms + _, perm = assert_matches_vstack(build, 6, rng) + assert list(perm) == [0, 2, 4, 6, 8, 1, 3, 5, 7, 9, 10, 11] + + +def test_shared_subexpression_and_parameter(rng): + def build(scope): + x = scope.Variable(5, 1) + p = scope.Parameter(1, 1, value=2.5) + terms = [] + for k in range(4): + d = x[k] - x[k + 1] # shared inside the term + terms.append(p * d * d + sp.tanh(d)) + return terms + assert_matches_vstack(build, 5, rng) + + +def test_templates_without_holes(rng): + def build(scope): + y = scope.Variable(1, 1) + return [y, y, sp.exp(y), sp.exp(y)] + assert_matches_vstack(build, 1, rng) From ea3937963a17c783afc39856eabcda4f4ad5c1b9 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 14:07:03 -0800 Subject: [PATCH 02/16] compact: use SLP/MergeFunctions naming (params, args, lanes) Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/_compact.py | 69 +++++++++++++++++++++++----------------- tests/test_compact.py | 2 +- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/sparsediffpy/_compact.py b/sparsediffpy/_compact.py index 1235b14..892457c 100644 --- a/sparsediffpy/_compact.py +++ b/sparsediffpy/_compact.py @@ -1,10 +1,13 @@ """Merge scalar terms written in a Python loop into vector expressions. A loop such as ``[x[k] ** 2 + x[k + 1] - 1 for k in range(n - 1)]`` gives the -engine one small expression per term. The terms share a *template*: the same -operations, differing only in which entries of a variable and which scalar -constants they use. :func:`compact` rebuilds each template once as a vector -expression, with the varying parts gathered into index arrays and constants. +engine one small expression per term. :func:`compact` vectorizes them in the +spirit of LLVM's SLP vectorizer: terms with the same *template* (isomorphic +trees) are bundled, each term is one *lane*, and the bundle is rebuilt once as +a vector expression. As in LLVM's MergeFunctions, the operands that differ +between terms -- the index of ``x[i]`` and scalar constants -- become the +template's *params*; each term supplies its own *args*, which are gathered into +an index array or a constant vector. """ import numpy as np @@ -38,31 +41,32 @@ def compact(terms): if t.shape != (1, 1): raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") try: - key, holes = _template(t, leaves) + key, args = _template(t, leaves) except _Unsupported: - key, holes = ("single", i), None - groups.setdefault(key, []).append((i, t, holes)) + key, args = ("single", i), None + groups.setdefault(key, []).append((i, t, args)) parts, perm = [], [] for key, members in groups.items(): perm += [i for i, _, _ in members] parts.append(members[0][1] if len(members) == 1 else - _rebuild(key, [holes for _, _, holes in members], leaves)) + _vectorize(key, [args for _, _, args in members], leaves)) expr = parts[0] if len(parts) == 1 else _atoms.vstack(parts) perm = None if perm == list(range(len(perm))) else np.array(perm) return expr, perm def _template(term, leaves): - """Describe a scalar term as ``(template, holes)``. - - The template lists the term's nodes in post-order. Each entry names the - operation and refers to its children by their position in the list, so a - subexpression shared inside the term is part of the template. Holes are - what varies between terms -- the entry index of ``x[i]`` and the value of a - scalar constant -- in the order they are met. + """Describe a scalar term as ``(template, args)``. + + The template lists the term's nodes in post-order, like instructions in + SSA form: each entry names the operation and refers to its operands by + their position in the list, so a subexpression shared inside the term is + part of the template. ``("gather", id(x))`` and ``("const",)`` entries are + params; ``args`` holds this term's value for each (the index ``i`` of + ``x[i]``, the constant's value), in template order. """ - entries, holes, seen = [], [], {} + entries, args, seen = [], [], {} def visit(e): if id(e) in seen: @@ -71,7 +75,7 @@ def visit(e): raise _Unsupported if isinstance(e, Constant): ent = ("const",) - holes.append(float(e.value[0, 0])) + args.append(float(e.value[0, 0])) elif isinstance(e, (Variable, Parameter)): leaves[id(e)] = e ent = ("leaf", id(e)) @@ -79,8 +83,8 @@ def visit(e): raise _Unsupported elif e.op == "index" and isinstance(e.args[0], Variable): leaves[id(e.args[0])] = e.args[0] - ent = ("var", id(e.args[0])) - holes.append(int(e.attrs["idx"][0])) + ent = ("gather", id(e.args[0])) + args.append(int(e.attrs["idx"][0])) elif e.op in ("add", "multiply", "scalar_mult"): ent = (e.op, visit(e.args[0]), visit(e.args[1])) elif e.op == "neg" or e.op in _UNARY: @@ -94,21 +98,26 @@ def visit(e): return seen[id(e)] visit(term) - return tuple(entries), holes + return tuple(entries), args + +def _vectorize(template, args, leaves): + """Evaluate the template once on (lanes, 1) vectors, one lane per term. -def _rebuild(template, holes, leaves): - """Evaluate the template once on (K, 1) vectors, one row per term.""" - K = len(holes) - columns = iter(zip(*holes)) # the K values of each hole, in template order + ``args[lane]`` is that term's args; transposed, each param gets one value + per lane. + """ + lanes = len(args) + params = iter(zip(*args)) vals = [] for ent in template: op = ent[0] - if op == "var": - vals.append(index_flat(leaves[ent[1]], next(columns), (K, 1))) + if op == "gather": + vals.append(index_flat(leaves[ent[1]], next(params), (lanes, 1))) elif op == "const": - c = np.array(next(columns)) - vals.append(Constant(c[0] if np.all(c == c[0]) else c.reshape(K, 1))) + c = np.array(next(params)) + vals.append(Constant(c[0] if np.all(c == c[0]) + else c.reshape(lanes, 1))) elif op == "leaf": vals.append(leaves[ent[1]]) elif op == "add": @@ -121,5 +130,5 @@ def _rebuild(template, holes, leaves): vals.append(vals[ent[2]] ** ent[1]) else: vals.append(getattr(_atoms, op)(vals[ent[1]])) - # a template without holes (every term is e.g. the scalar y) builds a scalar - return broadcast(vals[-1], (K, 1)) + # a template without params (every term is e.g. the scalar y) is a scalar + return broadcast(vals[-1], (lanes, 1)) diff --git a/tests/test_compact.py b/tests/test_compact.py index 44f5faa..ab6672e 100644 --- a/tests/test_compact.py +++ b/tests/test_compact.py @@ -76,7 +76,7 @@ def build(scope): assert_matches_vstack(build, 5, rng) -def test_templates_without_holes(rng): +def test_templates_without_params(rng): def build(scope): y = scope.Variable(1, 1) return [y, y, sp.exp(y), sp.exp(y)] From cc9663c6374cfed2c17ac4b97e8e4ed6d4438c94 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 14:12:44 -0800 Subject: [PATCH 03/16] Drop leading underscores from Python modules; LLVM IR naming in compact Modules: _atoms -> atoms, _expression -> expression, _compile -> problem (compile.py would clash with sp.compile), _compact -> vectorize. compact now lowers each term to (opcode, *operands) instructions whose operands are value numbers, plus per-term args for each load/const. Node.op is renamed Node.opcode. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +- sparsediffpy/__init__.py | 8 +- sparsediffpy/_compact.py | 134 ----------------- sparsediffpy/{_atoms.py => atoms.py} | 2 +- .../{_expression.py => expression.py} | 8 +- sparsediffpy/{_compile.py => problem.py} | 2 +- sparsediffpy/vectorize.py | 140 ++++++++++++++++++ tests/test_api.py | 8 +- tests/{test_compact.py => test_vectorize.py} | 8 +- 9 files changed, 161 insertions(+), 155 deletions(-) delete mode 100644 sparsediffpy/_compact.py rename sparsediffpy/{_atoms.py => atoms.py} (99%) rename sparsediffpy/{_expression.py => expression.py} (98%) rename sparsediffpy/{_compile.py => problem.py} (98%) create mode 100644 sparsediffpy/vectorize.py rename tests/{test_compact.py => test_vectorize.py} (92%) diff --git a/README.md b/README.md index 39687d4..99645b3 100644 --- a/README.md +++ b/README.md @@ -120,11 +120,11 @@ x = scope.Variable(n, 1) f, _ = sp.compact([(x[k] - 1.0) ** 2 for k in range(n)]) g, perm = sp.compact([x[k] ** 2 + x[k + 1] - 1.0 for k in range(n - 1)]) prob = sp.Problem(sp.sum(f), [g]) # 2 vector expressions, not ~2000 scalar ones -assert perm is None # one template per list, so rows kept their order +assert perm is None # each list merged into one group, so no row moved ``` -When a list mixes templates, rows are grouped by template: row `j` of the result -is `terms[perm[j]]`. Map row data such as bounds in with `values[perm]`, and +When a list mixes terms with different structure, rows are grouped by +structure: row `j` of the result is `terms[perm[j]]`. Map row data such as bounds in with `values[perm]`, and multipliers back out with `out[perm] = lam`. ## Supported operations diff --git a/sparsediffpy/__init__.py b/sparsediffpy/__init__.py index ab3e2c8..0a59ec1 100644 --- a/sparsediffpy/__init__.py +++ b/sparsediffpy/__init__.py @@ -15,13 +15,13 @@ """ from sparsediffpy import _sparsediffengine # noqa: F401 -from sparsediffpy._expression import ( # noqa: F401 +from sparsediffpy.expression import ( # noqa: F401 DimensionError, Expression, Parameter, Scope, Variable, broadcast, index_flat, ) -from sparsediffpy._atoms import ( # noqa: F401 +from sparsediffpy.atoms import ( # noqa: F401 asinh, atan, atan2, atanh, cos, diag_vec, entr, exp, hstack, log, logistic, normal_cdf, power, prod, quad_form, quad_over_lin, rel_entr, reshape, sin, sinh, sum, tan, tanh, trace, vstack, xexp, ) -from sparsediffpy._compile import CompiledExpression, Problem, compile # noqa: F401 -from sparsediffpy._compact import compact # noqa: F401 +from sparsediffpy.problem import CompiledExpression, Problem, compile # noqa: F401 +from sparsediffpy.vectorize import compact # noqa: F401 diff --git a/sparsediffpy/_compact.py b/sparsediffpy/_compact.py deleted file mode 100644 index 892457c..0000000 --- a/sparsediffpy/_compact.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Merge scalar terms written in a Python loop into vector expressions. - -A loop such as ``[x[k] ** 2 + x[k + 1] - 1 for k in range(n - 1)]`` gives the -engine one small expression per term. :func:`compact` vectorizes them in the -spirit of LLVM's SLP vectorizer: terms with the same *template* (isomorphic -trees) are bundled, each term is one *lane*, and the bundle is rebuilt once as -a vector expression. As in LLVM's MergeFunctions, the operands that differ -between terms -- the index of ``x[i]`` and scalar constants -- become the -template's *params*; each term supplies its own *args*, which are gathered into -an index array or a constant vector. -""" - -import numpy as np - -from sparsediffpy import _atoms -from sparsediffpy._expression import ( - Constant, Node, Parameter, Variable, cast_to_const, broadcast, index_flat, -) - -# Unary atoms that accept any argument (restricted ones need plain Variables). -_UNARY = {"sin", "cos", "exp", "sinh", "tanh", "asinh", "atan", "logistic", - "normal_cdf", "xexp"} - - -class _Unsupported(Exception): - pass - - -def compact(terms): - """Stack scalar expressions like :func:`vstack`, merging shared templates. - - Returns ``(expr, perm)``: ``expr`` has shape ``(len(terms), 1)`` and its - row ``j`` is ``terms[perm[j]]``, or ``perm`` is ``None`` if no row moved. - Terms using other operations (matmul, sum, ...) are kept as they are. - """ - if not terms: - raise ValueError("compact: need at least one term") - leaves, groups = {}, {} # leaves: id -> Variable/Parameter used by templates - for i, t in enumerate(terms): - t = cast_to_const(t) - if t.shape != (1, 1): - raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") - try: - key, args = _template(t, leaves) - except _Unsupported: - key, args = ("single", i), None - groups.setdefault(key, []).append((i, t, args)) - - parts, perm = [], [] - for key, members in groups.items(): - perm += [i for i, _, _ in members] - parts.append(members[0][1] if len(members) == 1 else - _vectorize(key, [args for _, _, args in members], leaves)) - expr = parts[0] if len(parts) == 1 else _atoms.vstack(parts) - perm = None if perm == list(range(len(perm))) else np.array(perm) - return expr, perm - - -def _template(term, leaves): - """Describe a scalar term as ``(template, args)``. - - The template lists the term's nodes in post-order, like instructions in - SSA form: each entry names the operation and refers to its operands by - their position in the list, so a subexpression shared inside the term is - part of the template. ``("gather", id(x))`` and ``("const",)`` entries are - params; ``args`` holds this term's value for each (the index ``i`` of - ``x[i]``, the constant's value), in template order. - """ - entries, args, seen = [], [], {} - - def visit(e): - if id(e) in seen: - return seen[id(e)] - if e.shape != (1, 1): - raise _Unsupported - if isinstance(e, Constant): - ent = ("const",) - args.append(float(e.value[0, 0])) - elif isinstance(e, (Variable, Parameter)): - leaves[id(e)] = e - ent = ("leaf", id(e)) - elif not isinstance(e, Node): - raise _Unsupported - elif e.op == "index" and isinstance(e.args[0], Variable): - leaves[id(e.args[0])] = e.args[0] - ent = ("gather", id(e.args[0])) - args.append(int(e.attrs["idx"][0])) - elif e.op in ("add", "multiply", "scalar_mult"): - ent = (e.op, visit(e.args[0]), visit(e.args[1])) - elif e.op == "neg" or e.op in _UNARY: - ent = (e.op, visit(e.args[0])) - elif e.op == "power": - ent = ("power", e.attrs["p"], visit(e.args[0])) - else: - raise _Unsupported - seen[id(e)] = len(entries) - entries.append(ent) - return seen[id(e)] - - visit(term) - return tuple(entries), args - - -def _vectorize(template, args, leaves): - """Evaluate the template once on (lanes, 1) vectors, one lane per term. - - ``args[lane]`` is that term's args; transposed, each param gets one value - per lane. - """ - lanes = len(args) - params = iter(zip(*args)) - vals = [] - for ent in template: - op = ent[0] - if op == "gather": - vals.append(index_flat(leaves[ent[1]], next(params), (lanes, 1))) - elif op == "const": - c = np.array(next(params)) - vals.append(Constant(c[0] if np.all(c == c[0]) - else c.reshape(lanes, 1))) - elif op == "leaf": - vals.append(leaves[ent[1]]) - elif op == "add": - vals.append(vals[ent[1]] + vals[ent[2]]) - elif op in ("multiply", "scalar_mult"): - vals.append(vals[ent[1]] * vals[ent[2]]) - elif op == "neg": - vals.append(-vals[ent[1]]) - elif op == "power": - vals.append(vals[ent[2]] ** ent[1]) - else: - vals.append(getattr(_atoms, op)(vals[ent[1]])) - # a template without params (every term is e.g. the scalar y) is a scalar - return broadcast(vals[-1], (lanes, 1)) diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/atoms.py similarity index 99% rename from sparsediffpy/_atoms.py rename to sparsediffpy/atoms.py index b732f43..b94a187 100644 --- a/sparsediffpy/_atoms.py +++ b/sparsediffpy/atoms.py @@ -4,7 +4,7 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy._expression import ( +from sparsediffpy.expression import ( Constant, Node, Parameter, Variable, _check_shape, _check_symmetric, cast_to_const, ) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/expression.py similarity index 98% rename from sparsediffpy/_expression.py rename to sparsediffpy/expression.py index 86e2b24..bb66e53 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/expression.py @@ -85,16 +85,16 @@ def __iter__(self): class Node(Expression): """An operation: ``make(*child_capsules)`` builds its C capsule. - ``op`` names the operation and ``attrs`` holds the parameters that rewrites - such as :func:`compact` read (``idx`` for index, ``p`` for power). + ``opcode`` names the operation and ``attrs`` holds the parameters that + rewrites such as :func:`compact` read (``idx`` for index, ``p`` for power). Compilation only uses ``make``. """ - def __init__(self, make, args, shape, op=None, attrs=None): + def __init__(self, make, args, shape, opcode=None, attrs=None): self.make = make self.args = tuple(args) self.shape = tuple(shape) - self.op = op + self.opcode = opcode self.attrs = attrs diff --git a/sparsediffpy/_compile.py b/sparsediffpy/problem.py similarity index 98% rename from sparsediffpy/_compile.py rename to sparsediffpy/problem.py index 8554b10..913e489 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/problem.py @@ -10,7 +10,7 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy._expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const +from sparsediffpy.expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const def _find_scope(exprs): diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py new file mode 100644 index 0000000..3bdfee7 --- /dev/null +++ b/sparsediffpy/vectorize.py @@ -0,0 +1,140 @@ +"""Merge scalar terms written in a Python loop into vector expressions. + +A loop such as ``[x[k] ** 2 + x[k + 1] - 1 for k in range(n - 1)]`` gives the +engine one small expression per term. :func:`compact` lowers each term to a +short list of instructions, in the style of LLVM IR:: + + %0 = load x ; arg: the index k + %1 = power 2.0 %0 + %2 = load x ; arg: k + 1 + %3 = add %1 %2 + %4 = const ; arg: 1.0 + %5 = neg %4 + %6 = add %3 %5 + +Terms whose instructions are identical differ only in their args, so each such +group is rebuilt once as a vector expression: a ``load`` becomes a gather of +``x`` at the args, a ``const`` becomes a constant vector. +""" + +import numpy as np + +from sparsediffpy import atoms +from sparsediffpy.expression import ( + Constant, Node, Parameter, Variable, cast_to_const, broadcast, index_flat, +) + +# Unary atoms that accept any argument (restricted ones need plain Variables). +_UNARY = {"sin", "cos", "exp", "sinh", "tanh", "asinh", "atan", "logistic", + "normal_cdf", "xexp"} + + +class _Unsupported(Exception): + pass + + +def compact(terms): + """Stack scalar expressions like :func:`vstack`, merging identical ones. + + Returns ``(expr, perm)``: ``expr`` has shape ``(len(terms), 1)`` and its + row ``j`` is ``terms[perm[j]]``, or ``perm`` is ``None`` if no row moved. + Terms using other operations (matmul, sum, ...) are kept as they are. + """ + if not terms: + raise ValueError("compact: need at least one term") + values, groups = {}, {} # values: id -> Variable/Parameter the terms use + for i, t in enumerate(terms): + t = cast_to_const(t) + if t.shape != (1, 1): + raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") + try: + insts, args = _lower(t, values) + except _Unsupported: + insts, args = ("single", i), None + groups.setdefault(insts, []).append((i, t, args)) + + parts, perm = [], [] + for insts, members in groups.items(): + perm += [i for i, _, _ in members] + parts.append(members[0][1] if len(members) == 1 else + _vectorize(insts, [args for _, _, args in members], values)) + expr = parts[0] if len(parts) == 1 else atoms.vstack(parts) + perm = None if perm == list(range(len(perm))) else np.array(perm) + return expr, perm + + +def _lower(term, values): + """Lower a scalar term to ``(insts, args)``. + + ``insts`` lists the term's nodes in post-order as ``(opcode, *operands)``; + operands are value numbers (positions in the list), so a subexpression + shared inside the term is computed once. ``args`` holds the per-term data, + in instruction order: the index of each ``load`` and the value of each + ``const``. + """ + insts, args, number = [], [], {} # number: id(node) -> value number + + def visit(e): + if id(e) in number: + return number[id(e)] + if e.shape != (1, 1): + raise _Unsupported + if isinstance(e, Constant): + inst = ("const",) + args.append(float(e.value[0, 0])) + elif isinstance(e, (Variable, Parameter)): + values[id(e)] = e + inst = ("global", id(e)) + elif not isinstance(e, Node): + raise _Unsupported + elif e.opcode == "index" and isinstance(e.args[0], Variable): + values[id(e.args[0])] = e.args[0] + inst = ("load", id(e.args[0])) + args.append(int(e.attrs["idx"][0])) + elif e.opcode in ("add", "multiply", "scalar_mult"): + inst = (e.opcode, visit(e.args[0]), visit(e.args[1])) + elif e.opcode == "neg" or e.opcode in _UNARY: + inst = (e.opcode, visit(e.args[0])) + elif e.opcode == "power": + inst = ("power", e.attrs["p"], visit(e.args[0])) + else: + raise _Unsupported + number[id(e)] = len(insts) + insts.append(inst) + return number[id(e)] + + visit(term) + return tuple(insts), args + + +def _vectorize(insts, args, values): + """Run ``insts`` once on (lanes, 1) vectors, one lane per term. + + ``args[lane]`` is that term's args; transposed, each ``load``/``const`` + gets one value per lane. + """ + lanes = len(args) + lane_args = iter(zip(*args)) + vecs = [] # vecs[n]: the vector for value number n + for inst in insts: + opcode = inst[0] + if opcode == "load": + vecs.append(index_flat(values[inst[1]], next(lane_args), (lanes, 1))) + elif opcode == "const": + c = np.array(next(lane_args)) + vecs.append(Constant(c[0] if np.all(c == c[0]) + else c.reshape(lanes, 1))) + elif opcode == "global": + vecs.append(values[inst[1]]) + elif opcode == "add": + vecs.append(vecs[inst[1]] + vecs[inst[2]]) + elif opcode in ("multiply", "scalar_mult"): + vecs.append(vecs[inst[1]] * vecs[inst[2]]) + elif opcode == "neg": + vecs.append(-vecs[inst[1]]) + elif opcode == "power": + vecs.append(vecs[inst[2]] ** inst[1]) + else: + vecs.append(getattr(atoms, opcode)(vecs[inst[1]])) + # with no load/const (every term is e.g. the scalar y) the result is a scalar + return broadcast(vecs[-1], (lanes, 1)) diff --git a/tests/test_api.py b/tests/test_api.py index ec645e6..00e879a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -6,7 +6,7 @@ import pytest import sparsediffpy as sp -from sparsediffpy import _compile +from sparsediffpy import problem # --- scope and values --------------------------------------------------------- @@ -278,13 +278,13 @@ def test_unset_parameter_raises(scope): # --- Caching ------------------------------------------------------------------ def count_calls(monkeypatch, name): - """Count calls to the engine function `name` as seen from _compile.""" - calls, real = [], getattr(_compile._C, name) + """Count calls to the engine function `name` as seen from sparsediffpy.problem.""" + calls, real = [], getattr(problem._C, name) def counted(*args): calls.append(1) return real(*args) - monkeypatch.setattr(_compile._C, name, counted) + monkeypatch.setattr(problem._C, name, counted) return calls diff --git a/tests/test_compact.py b/tests/test_vectorize.py similarity index 92% rename from tests/test_compact.py rename to tests/test_vectorize.py index ab6672e..52e8880 100644 --- a/tests/test_compact.py +++ b/tests/test_vectorize.py @@ -4,7 +4,7 @@ import scipy.sparse as sps import sparsediffpy as sp -from sparsediffpy._expression import Constant +from sparsediffpy.expression import Constant def _evaluate(g, u, lam): @@ -38,7 +38,7 @@ def build(scope): return [3 * x[k + 1] ** 3 + 2 * x[k + 2] - x[k] * sp.exp(x[k] - x[k + 1]) for k in range(6)] g, perm = assert_matches_vstack(build, 8, rng) - assert perm is None and g.op != "vstack" # a single template, in place + assert perm is None and g.opcode != "vstack" # one instruction list, rows in place def test_varying_constants(rng): @@ -48,7 +48,7 @@ def build(scope): assert_matches_vstack(build, 5, rng) -def test_interleaved_templates_and_unsupported_terms(rng): +def test_interleaved_groups_and_unsupported_terms(rng): A = rng.standard_normal((2, 6)) def build(scope): @@ -76,7 +76,7 @@ def build(scope): assert_matches_vstack(build, 5, rng) -def test_templates_without_params(rng): +def test_terms_without_args(rng): def build(scope): y = scope.Variable(1, 1) return [y, y, sp.exp(y), sp.exp(y)] From afc454ffdac86c4868c0cb794a86da1a163f8ef6 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 14:21:41 -0800 Subject: [PATCH 04/16] Drop the README section on compact Co-Authored-By: Claude Opus 5.5 --- README.md | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/README.md b/README.md index 99645b3..b11e3b7 100644 --- a/README.md +++ b/README.md @@ -101,32 +101,6 @@ res = minimize(prob.objective_forward, np.zeros(4), jac=grad, hess=obj_hess, print(res.x) # approx. [0. 0.289 0.549 0.784], on the boundary ||x|| = 1 ``` -## Models written as loops - -A model built term by term in a Python loop gives the engine one small scalar -expression per term, which is much slower than the same model written with -vectors. `sp.compact(terms)` is a drop-in for `sp.vstack(terms)` that merges -terms with the same structure (differing only in which entries and constants -they use) into one vector expression: - -```python -import numpy as np - -import sparsediffpy as sp - -n = 1000 -scope = sp.Scope() -x = scope.Variable(n, 1) -f, _ = sp.compact([(x[k] - 1.0) ** 2 for k in range(n)]) -g, perm = sp.compact([x[k] ** 2 + x[k + 1] - 1.0 for k in range(n - 1)]) -prob = sp.Problem(sp.sum(f), [g]) # 2 vector expressions, not ~2000 scalar ones -assert perm is None # each list merged into one group, so no row moved -``` - -When a list mixes terms with different structure, rows are grouped by -structure: row `j` of the result is `terms[perm[j]]`. Map row data such as bounds in with `values[perm]`, and -multipliers back out with `out[perm] = lam`. - ## Supported operations - **Operators:** `+ - * @ **`, `.T`, and indexing with ints, slices and integer lists: `x[1:3]` on a From 926a66aa6eefaa0bbbb1374a41a05384c8b3e342 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 14:29:09 -0800 Subject: [PATCH 05/16] test_vectorize: check group counts; cover non-merging, Parameter updates, errors Co-Authored-By: Claude Opus 5.5 --- tests/test_vectorize.py | 56 +++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/tests/test_vectorize.py b/tests/test_vectorize.py index 52e8880..1b44b2b 100644 --- a/tests/test_vectorize.py +++ b/tests/test_vectorize.py @@ -1,6 +1,7 @@ -"""compact(terms) must match vstack(terms) in value and derivatives.""" +"""compact(terms) must merge what it can and match vstack(terms) exactly.""" import numpy as np +import pytest import scipy.sparse as sps import sparsediffpy as sp @@ -15,21 +16,28 @@ def _evaluate(g, u, lam): prob.hessian(0.7, lam).toarray()) -def assert_matches_vstack(build, n_vars, rng): - """build(scope) -> list of scalar terms; compare compact with vstack.""" +def _n_groups(g): + # compact returns one part per group, stacked only if there are several + return len(g.args) if g.opcode == "vstack" else 1 + + +def assert_compact(build, n_vars, rng, groups): + """build(scope) -> scalar terms. Check the group count, then compare value, + gradient, Jacobian and Hessian with vstack(terms), mapping rows by perm.""" terms = build(sp.Scope()) u = rng.uniform(0.5, 1.5, n_vars) lam = rng.standard_normal(len(terms)) f, grad, c, J, H = _evaluate(sp.vstack(terms), u, lam) g, perm = sp.compact(build(sp.Scope())) + assert _n_groups(g) == groups rows = slice(None) if perm is None else perm got = _evaluate(g, u, lam[rows]) expected = (f, grad, c[rows], J[rows], H) for name, a, b in zip(["objective", "gradient", "constraints", "jacobian", "hessian"], expected, got): np.testing.assert_allclose(b, a, rtol=1e-12, atol=1e-12, err_msg=name) - return g, perm + return perm def test_loop_merges_into_one_vector(rng): @@ -37,15 +45,14 @@ def build(scope): x = scope.Variable(8, 1) return [3 * x[k + 1] ** 3 + 2 * x[k + 2] - x[k] * sp.exp(x[k] - x[k + 1]) for k in range(6)] - g, perm = assert_matches_vstack(build, 8, rng) - assert perm is None and g.opcode != "vstack" # one instruction list, rows in place + assert assert_compact(build, 8, rng, groups=1) is None # rows in place def test_varying_constants(rng): def build(scope): x = scope.Variable(5, 1) return [(k + 1.0) * (x[k] - k) ** 2 for k in range(5)] - assert_matches_vstack(build, 5, rng) + assert_compact(build, 5, rng, groups=1) def test_interleaved_groups_and_unsupported_terms(rng): @@ -60,7 +67,7 @@ def build(scope): terms += [sp.exp(Constant(A[:1]) @ x), sp.exp(Constant(sps.csr_matrix(A[1:])) @ x)] return terms - _, perm = assert_matches_vstack(build, 6, rng) + perm = assert_compact(build, 6, rng, groups=4) assert list(perm) == [0, 2, 4, 6, 8, 1, 3, 5, 7, 9, 10, 11] @@ -73,11 +80,40 @@ def build(scope): d = x[k] - x[k + 1] # shared inside the term terms.append(p * d * d + sp.tanh(d)) return terms - assert_matches_vstack(build, 5, rng) + assert_compact(build, 5, rng, groups=1) def test_terms_without_args(rng): def build(scope): y = scope.Variable(1, 1) return [y, y, sp.exp(y), sp.exp(y)] - assert_matches_vstack(build, 1, rng) + assert_compact(build, 1, rng, groups=2) + + +def test_different_structure_does_not_merge(rng): + # different variable, exponent or sharing pattern: separate groups + def build(scope): + x, z = scope.Variable(3, 1), scope.Variable(3, 1) + return [x[0] ** 2, x[1] ** 2, z[0] ** 2, x[2] ** 3, + x[0] * x[1], x[1] * x[1]] + assert_compact(build, 6, rng, groups=4) + + +def test_parameter_update_after_compact(): + scope = sp.Scope() + x = scope.Variable(3, 1) + p = scope.Parameter(1, 1, value=2.0) + g, _ = sp.compact([p * x[k] for k in range(3)]) + prob = sp.Problem(sp.sum(g), [g]) + u = np.array([1.0, 2.0, 3.0]) + np.testing.assert_allclose(prob.constraint_forward(u), 2.0 * u) + p.value = 5.0 + np.testing.assert_allclose(prob.constraint_forward(u), 5.0 * u) + + +def test_rejects_non_scalar_and_empty(): + x = sp.Scope().Variable(3, 1) + with pytest.raises(ValueError): + sp.compact([x[0], x]) + with pytest.raises(ValueError): + sp.compact([]) From 8a739c33f8aee5ea3d97167efa8d65b196e55637 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 14:41:35 -0800 Subject: [PATCH 06/16] Test compact on benchmark NLPs (Luksan-Vlcek, Brusselator, logreg, VdP RK4) Each problem is written vectorized and as a scalar loop; the loop form is checked through vstack and compact against the vectorized reference (f, grad, g, J, Lagrangian Hessian), with group counts pinned. Co-Authored-By: Claude Opus 5.5 --- tests/test_vectorize_problems.py | 354 +++++++++++++++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 tests/test_vectorize_problems.py diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py new file mode 100644 index 0000000..8877767 --- /dev/null +++ b/tests/test_vectorize_problems.py @@ -0,0 +1,354 @@ +"""compact() on the benchmark NLPs (Luksan-Vlcek, Brusselator, logistic +regression, Van der Pol RK4 shooting). + +Each problem is written twice with identical math, as in the paper's +experiments: ``vec`` (slices, gathers, matmul) and ``scalar`` (a Python loop +producing one scalar term per row, JuMP/Pyomo style). For the scalar form we +build the objective and constraints three ways and require the NLP oracles +f, grad f, g, J_g and the Lagrangian Hessian to agree: + + * vec - the hand-vectorized formulation (independent reference) + * vstack - scalar terms stacked as they are + * compact - scalar terms merged by sp.compact, rows mapped back via perm + +We also pin down how many groups compact finds, so a regression that stops +merging (and silently falls back to one node per term) is caught. +""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +import sparsediffpy as sp +from sparsediffpy.expression import Constant + +TOL = dict(rtol=1e-10, atol=1e-10) + + +# ------------------------------------------------------------------ problems + + +def luksan_data(n): + return {"n": n} + + +def luksan_vec(x, d, ops): + n = d["n"] + x0, x1 = x[0:n - 1], x[1:n] + f = ops.sum(100.0 * (x0 ** 2 - x1) ** 2 + (x0 - 1.0) ** 2) + a, b, c = x[0:n - 2], x[1:n - 1], x[2:n] + g = (3.0 * b ** 3 + 2.0 * c - 5.0 + ops.sin(b - c) * ops.sin(b + c) + + 4.0 * b - a * ops.exp(a - b) - 3.0) + return f, [g] + + +def luksan_scalar(x, d, ops): + n = d["n"] + xs = [x[i] for i in range(n)] + f = [100.0 * (xs[i] ** 2 - xs[i + 1]) ** 2 + (xs[i] - 1.0) ** 2 + for i in range(n - 1)] + g = [3.0 * xs[k + 1] ** 3 + 2.0 * xs[k + 2] - 5.0 + + ops.sin(xs[k + 1] - xs[k + 2]) * ops.sin(xs[k + 1] + xs[k + 2]) + + 4.0 * xs[k + 1] - xs[k] * ops.exp(xs[k] - xs[k + 1]) - 3.0 + for k in range(n - 2)] + return f, g + + +def brusselator_data(N): + idx = np.arange(N * N).reshape(N, N) + xs = np.linspace(0, 1, N) + X, Y = np.meshgrid(xs, xs, indexing="ij") + # a forcing that is nonzero on some cells only: a const that varies by lane + forcing = (((X - 0.3) ** 2 + (Y - 0.6) ** 2) <= 0.2 ** 2).astype(float) * 5.0 + return {"N": N, "n": 2 * N * N, "alpha": 10.0 * (N - 1) ** 2 / 1e4, + "forcing": forcing.ravel(), + "ip": np.roll(idx, -1, axis=0).ravel(), + "im": np.roll(idx, 1, axis=0).ravel(), + "jp": np.roll(idx, -1, axis=1).ravel(), + "jm": np.roll(idx, 1, axis=1).ravel()} + + +def brusselator_vec(x, d, ops): + M = d["N"] ** 2 + u, v = x[0:M], x[M:2 * M] + + def lap(w): + return (w[d["ip"]] + w[d["im"]] + w[d["jp"]] + w[d["jm"]] - 4.0 * w) + + u2v = u ** 2 * v + fu = (1.0 + u2v - 4.4 * u + d["alpha"] * lap(u) + + d["forcing"].reshape(-1, 1)) + fv = 3.4 * u - u2v + d["alpha"] * lap(v) + return None, [sp.vstack([fu, fv])] + + +def _brusselator_rows(x, d): + M = d["N"] ** 2 + xs = [x[i] for i in range(2 * M)] + u, v = xs[:M], xs[M:] + a, F = d["alpha"], d["forcing"] + ip, im, jp, jm = d["ip"], d["im"], d["jp"], d["jm"] + fu = [1.0 + u[k] ** 2 * v[k] - 4.4 * u[k] + + a * (u[ip[k]] + u[im[k]] + u[jp[k]] + u[jm[k]] - 4.0 * u[k]) + F[k] + for k in range(M)] + fv = [3.4 * u[k] - u[k] ** 2 * v[k] + + a * (v[ip[k]] + v[im[k]] + v[jp[k]] + v[jm[k]] - 4.0 * v[k]) + for k in range(M)] + return fu, fv + + +def brusselator_scalar(x, d, ops): + fu, fv = _brusselator_rows(x, d) + return None, fu + fv + + +def brusselator_interleaved(x, d, ops): + # same rows, written per grid cell: (fu_k, fv_k) for each k + fu, fv = _brusselator_rows(x, d) + return None, [t for pair in zip(fu, fv) for t in pair] + + +def logreg_data(n, seed=0): + rng = np.random.default_rng(seed) + m = 2 * n + A = rng.standard_normal((m, n)) / np.sqrt(n) + y = (rng.random(m) < 0.5).astype(float) + return {"n": n, "m": m, "A": A, "c": A.T @ y, "mu": 1e-2} + + +def logreg_vec(x, d, ops): + n = d["n"] + f = (ops.sum(ops.softplus(Constant(d["A"]) @ x)) + - Constant(d["c"].reshape(1, -1)) @ x + + 0.5 * d["mu"] * ops.sum(x ** 2)) + return f, [x[0:n - 1] ** 2 + x[1:n] ** 2] + + +def logreg_scalar(x, d, ops): + # the row dot products are matmuls: compact keeps those terms as they are + n, m, A = d["n"], d["m"], d["A"] + xs = [x[i] for i in range(n)] + f = [ops.softplus(Constant(A[i].reshape(1, -1)) @ x) for i in range(m)] + f += [-d["c"][j] * xs[j] + 0.5 * d["mu"] * xs[j] ** 2 for j in range(n)] + return f, [xs[i] ** 2 + xs[i + 1] ** 2 for i in range(n - 1)] + + +def vdp_data(N): + return {"N": N, "n": 3 * N + 2, "h": 10.0 / N} + + +def _vdp_rhs(a, b, u): + return (1.0 - b ** 2) * a - b + u, a + + +def _vdp_step(a, b, u, h): + k1a, k1b = _vdp_rhs(a, b, u) + k2a, k2b = _vdp_rhs(a + 0.5 * h * k1a, b + 0.5 * h * k1b, u) + k3a, k3b = _vdp_rhs(a + 0.5 * h * k2a, b + 0.5 * h * k2b, u) + k4a, k4b = _vdp_rhs(a + h * k3a, b + h * k3b, u) + return (a + (h / 6.0) * (k1a + 2.0 * k2a + 2.0 * k3a + k4a), + b + (h / 6.0) * (k1b + 2.0 * k2b + 2.0 * k3b + k4b)) + + +def vdp_vec(x, d, ops): + N, h = d["N"], d["h"] + x1, x2, u = x[0:N + 1], x[N + 1:2 * N + 2], x[2 * N + 2:3 * N + 2] + f = ops.sum(x1 ** 2) + ops.sum(x2 ** 2) + ops.sum(u ** 2) + na, nb = _vdp_step(x1[0:N], x2[0:N], u, h) + return f, [sp.vstack([x1[1:N + 1] - na, x2[1:N + 1] - nb])] + + +def vdp_scalar(x, d, ops): + # deep terms with heavy sharing: x1[k], x2[k], u[k] and the RK stages + # are reused many times inside each defect + N, h = d["N"], d["h"] + xs = [x[i] for i in range(3 * N + 2)] + x1, x2, u = xs[0:N + 1], xs[N + 1:2 * N + 2], xs[2 * N + 2:] + g1, g2 = [], [] + for k in range(N): + na, nb = _vdp_step(x1[k], x2[k], u[k], h) + g1.append(x1[k + 1] - na) + g2.append(x2[k + 1] - nb) + return [t ** 2 for t in xs], g1 + g2 + + +OPS = SimpleNamespace(sum=sp.sum, sin=sp.sin, exp=sp.exp, softplus=sp.logistic) + +# name: (data, vec, scalar, size, objective groups, constraint groups) +PROBLEMS = { + "luksan": (luksan_data, luksan_vec, luksan_scalar, 12, 1, 1), + "brusselator": (brusselator_data, brusselator_vec, brusselator_scalar, + 4, None, 2), + "logreg": (logreg_data, logreg_vec, logreg_scalar, 6, 2 * 6 + 1, 1), + "vdp": (vdp_data, vdp_vec, vdp_scalar, 6, 1, 2), +} + + +# ------------------------------------------------------------------ harness + + +def _n_groups(g): + return len(g.args) if g.opcode == "vstack" else 1 + + +def _oracles(f, g, u, sigma, lam): + prob = sp.Problem(0.0 if f is None else f, [g]) + fv = prob.objective_forward(u) + c = np.asarray(prob.constraint_forward(u)).ravel().copy() + grad = np.asarray(prob.gradient()).ravel().copy() + J = prob.jacobian().toarray() + H = prob.hessian(sigma, lam).toarray() + return float(fv), grad, c, J, H + + +def _build(form, d, compact): + """Objective and stacked constraints; perm maps compact rows to terms.""" + x = sp.Scope().Variable(d["n"], 1) + f, g = form(x, d, OPS) + if isinstance(f, list): # scalar objective terms: sum of a stack + f = sp.sum(sp.compact(f)[0] if compact else sp.vstack(f)) + if not compact: + return f, sp.vstack(g) if len(g) > 1 else g[0], None + gc, perm = sp.compact(g) + return f, gc, perm + + +def _point(d, m, seed=1): + rng = np.random.default_rng(seed) + return 0.5 + 0.3 * rng.standard_normal(d["n"]), 1.3, rng.standard_normal(m) + + +def _assert_same(got, expected, perm=None): + rows = slice(None) if perm is None else perm + exp_f, exp_grad, exp_c, exp_J, exp_H = expected + for name, a, b in zip( + ["objective", "gradient", "constraints", "jacobian", "hessian"], + got, (exp_f, exp_grad, exp_c[rows], exp_J[rows], exp_H)): + np.testing.assert_allclose(a, b, err_msg=name, **TOL) + + +# ------------------------------------------------------------------ tests + + +@pytest.mark.parametrize("name", PROBLEMS) +def test_compact_matches_vstack_and_vec(name): + data, vec, scalar, size, f_groups, g_groups = PROBLEMS[name] + d = data(size) + + f_vec, g_vec, _ = _build(vec, d, compact=False) + f_vs, g_vs, _ = _build(scalar, d, compact=False) + f_c, g_c, perm = _build(scalar, d, compact=True) + m = g_vec.shape[0] + assert g_vs.shape == g_c.shape == (m, 1) + if perm is not None: + assert sorted(perm) == list(range(m)) + + u, sigma, lam = _point(d, m) + ref = _oracles(f_vec, g_vec, u, sigma, lam) + # the scalar form is the same math as the vectorized one ... + _assert_same(_oracles(f_vs, g_vs, u, sigma, lam), ref) + # ... and compact keeps it, with multipliers given in compact's row order + lam_c = lam if perm is None else lam[perm] + _assert_same(_oracles(f_c, g_c, u, sigma, lam_c), ref, perm) + + assert _n_groups(g_c) == g_groups + if f_groups is not None: + assert _n_groups(f_c.args[0]) == f_groups + + +@pytest.mark.parametrize("name", PROBLEMS) +def test_compact_at_several_points(name): + # forward/derivative caches must not leak between evaluations + data, vec, scalar, size, _, _ = PROBLEMS[name] + d = data(size) + f_vec, g_vec, _ = _build(vec, d, compact=False) + f_c, g_c, perm = _build(scalar, d, compact=True) + p_vec = sp.Problem(0.0 if f_vec is None else f_vec, [g_vec]) + p_c = sp.Problem(0.0 if f_c is None else f_c, [g_c]) + rows = slice(None) if perm is None else perm + for seed in range(4): + u, sigma, lam = _point(d, g_vec.shape[0], seed) + np.testing.assert_allclose(p_c.objective_forward(u), + p_vec.objective_forward(u), **TOL) + np.testing.assert_allclose(p_c.gradient(), p_vec.gradient(), **TOL) + np.testing.assert_allclose( + np.asarray(p_c.constraint_forward(u)).ravel(), + np.asarray(p_vec.constraint_forward(u)).ravel()[rows], **TOL) + np.testing.assert_allclose(p_c.jacobian().toarray(), + p_vec.jacobian().toarray()[rows], **TOL) + np.testing.assert_allclose(p_c.hessian(sigma, lam[rows]).toarray(), + p_vec.hessian(sigma, lam).toarray(), **TOL) + + +def test_brusselator_interleaved_rows_get_permuted(): + d = brusselator_data(3) + M = 9 + x = sp.Scope().Variable(d["n"], 1) + _, terms = brusselator_interleaved(x, d, OPS) + g_c, perm = sp.compact(terms) + assert _n_groups(g_c) == 2 + # all fu rows (even positions) first, then all fv rows + assert list(perm) == list(range(0, 2 * M, 2)) + list(range(1, 2 * M, 2)) + + _, g_vec, _ = _build(brusselator_vec, d, compact=False) + u, sigma, lam = _point(d, 2 * M) + # interleaved row r is vec row (r // 2) + M * (r % 2) + to_vec = np.array([r // 2 + M * (r % 2) for r in range(2 * M)]) + ref = _oracles(None, g_vec, u, sigma, lam) + got = _oracles(None, g_c, u, sigma, lam[to_vec[perm]]) + _assert_same(got, ref, to_vec[perm]) + + +def test_brusselator_varying_forcing_is_a_vector_constant(): + # forcing is 0 on most cells and 5 on some: the merged const must keep + # per-lane values rather than collapse to the first lane's + d = brusselator_data(4) + assert 0 < np.count_nonzero(d["forcing"]) < d["forcing"].size + d0 = dict(d, forcing=np.zeros_like(d["forcing"])) + u, _, _ = _point(d, 2 * 16) + _, g, _ = _build(brusselator_scalar, d, compact=True) + _, g0, _ = _build(brusselator_scalar, d0, compact=True) + c = sp.Problem(0.0, [g]).constraint_forward(u) + c0 = sp.Problem(0.0, [g0]).constraint_forward(u) + np.testing.assert_allclose(np.asarray(c - c0).ravel()[:16], d["forcing"], + **TOL) + + +@pytest.mark.parametrize("N", [1, 2, 20]) +def test_vdp_horizon_lengths(N): + # a one-stage horizon (one lane per group) up to a longer one + d = vdp_data(N) + f_vec, g_vec, _ = _build(vdp_vec, d, compact=False) + f_c, g_c, perm = _build(vdp_scalar, d, compact=True) + assert perm is None + u, sigma, lam = _point(d, 2 * N) + _assert_same(_oracles(f_c, g_c, u, sigma, lam), + _oracles(f_vec, g_vec, u, sigma, lam)) + + +@pytest.mark.parametrize("n", [3, 4, 50]) +def test_luksan_sizes(n): + # n = 3 leaves a single constraint term: nothing to merge it with + d = luksan_data(n) + f_vec, g_vec, _ = _build(luksan_vec, d, compact=False) + f_c, g_c, perm = _build(luksan_scalar, d, compact=True) + assert perm is None + u, sigma, lam = _point(d, n - 2) + _assert_same(_oracles(f_c, g_c, u, sigma, lam), + _oracles(f_vec, g_vec, u, sigma, lam)) + + +def test_compact_shrinks_the_graph(): + # the point of compact: far fewer nodes than one subtree per term + def count(e, seen=None): + seen = set() if seen is None else seen + if id(e) in seen: + return 0 + seen.add(id(e)) + return 1 + sum(count(a, seen) for a in getattr(e, "args", ())) + + d = luksan_data(200) + _, g_vs, _ = _build(luksan_scalar, d, compact=False) + _, g_c, _ = _build(luksan_scalar, d, compact=True) + assert count(g_c) * 50 < count(g_vs) From 99b65cd656d9c0de48918af2a6d2b94398ac5216 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 14:47:23 -0800 Subject: [PATCH 07/16] Tidy test_vectorize_problems: closures per problem, one harness Co-Authored-By: Claude Opus 5.5 --- tests/test_vectorize_problems.py | 446 ++++++++++--------------------- 1 file changed, 143 insertions(+), 303 deletions(-) diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py index 8877767..2b265ef 100644 --- a/tests/test_vectorize_problems.py +++ b/tests/test_vectorize_problems.py @@ -1,21 +1,4 @@ -"""compact() on the benchmark NLPs (Luksan-Vlcek, Brusselator, logistic -regression, Van der Pol RK4 shooting). - -Each problem is written twice with identical math, as in the paper's -experiments: ``vec`` (slices, gathers, matmul) and ``scalar`` (a Python loop -producing one scalar term per row, JuMP/Pyomo style). For the scalar form we -build the objective and constraints three ways and require the NLP oracles -f, grad f, g, J_g and the Lagrangian Hessian to agree: - - * vec - the hand-vectorized formulation (independent reference) - * vstack - scalar terms stacked as they are - * compact - scalar terms merged by sp.compact, rows mapped back via perm - -We also pin down how many groups compact finds, so a regression that stops -merging (and silently falls back to one node per term) is caught. -""" - -from types import SimpleNamespace +"""compact() on benchmark NLPs must match the hand-vectorized formulation.""" import numpy as np import pytest @@ -26,329 +9,186 @@ TOL = dict(rtol=1e-10, atol=1e-10) -# ------------------------------------------------------------------ problems - - -def luksan_data(n): - return {"n": n} - - -def luksan_vec(x, d, ops): - n = d["n"] - x0, x1 = x[0:n - 1], x[1:n] - f = ops.sum(100.0 * (x0 ** 2 - x1) ** 2 + (x0 - 1.0) ** 2) - a, b, c = x[0:n - 2], x[1:n - 1], x[2:n] - g = (3.0 * b ** 3 + 2.0 * c - 5.0 + ops.sin(b - c) * ops.sin(b + c) - + 4.0 * b - a * ops.exp(a - b) - 3.0) - return f, [g] - - -def luksan_scalar(x, d, ops): - n = d["n"] - xs = [x[i] for i in range(n)] - f = [100.0 * (xs[i] ** 2 - xs[i + 1]) ** 2 + (xs[i] - 1.0) ** 2 - for i in range(n - 1)] - g = [3.0 * xs[k + 1] ** 3 + 2.0 * xs[k + 2] - 5.0 - + ops.sin(xs[k + 1] - xs[k + 2]) * ops.sin(xs[k + 1] + xs[k + 2]) - + 4.0 * xs[k + 1] - xs[k] * ops.exp(xs[k] - xs[k + 1]) - 3.0 - for k in range(n - 2)] - return f, g - - -def brusselator_data(N): - idx = np.arange(N * N).reshape(N, N) - xs = np.linspace(0, 1, N) - X, Y = np.meshgrid(xs, xs, indexing="ij") - # a forcing that is nonzero on some cells only: a const that varies by lane - forcing = (((X - 0.3) ** 2 + (Y - 0.6) ** 2) <= 0.2 ** 2).astype(float) * 5.0 - return {"N": N, "n": 2 * N * N, "alpha": 10.0 * (N - 1) ** 2 / 1e4, - "forcing": forcing.ravel(), - "ip": np.roll(idx, -1, axis=0).ravel(), - "im": np.roll(idx, 1, axis=0).ravel(), - "jp": np.roll(idx, -1, axis=1).ravel(), - "jm": np.roll(idx, 1, axis=1).ravel()} - +def luksan(n): + def vec(x): + x0, x1 = x[0:n - 1], x[1:n] + f = sp.sum(100.0 * (x0 ** 2 - x1) ** 2 + (x0 - 1.0) ** 2) + a, b, c = x[0:n - 2], x[1:n - 1], x[2:n] + g = (3.0 * b ** 3 + 2.0 * c - 5.0 + sp.sin(b - c) * sp.sin(b + c) + + 4.0 * b - a * sp.exp(a - b) - 3.0) + return f, g -def brusselator_vec(x, d, ops): - M = d["N"] ** 2 - u, v = x[0:M], x[M:2 * M] + def scalar(x): + xs = [x[i] for i in range(n)] + f = [100.0 * (xs[i] ** 2 - xs[i + 1]) ** 2 + (xs[i] - 1.0) ** 2 + for i in range(n - 1)] + g = [3.0 * xs[k + 1] ** 3 + 2.0 * xs[k + 2] - 5.0 + + sp.sin(xs[k + 1] - xs[k + 2]) * sp.sin(xs[k + 1] + xs[k + 2]) + + 4.0 * xs[k + 1] - xs[k] * sp.exp(xs[k] - xs[k + 1]) - 3.0 + for k in range(n - 2)] + return f, g - def lap(w): - return (w[d["ip"]] + w[d["im"]] + w[d["jp"]] + w[d["jm"]] - 4.0 * w) + return n, vec, scalar - u2v = u ** 2 * v - fu = (1.0 + u2v - 4.4 * u + d["alpha"] * lap(u) - + d["forcing"].reshape(-1, 1)) - fv = 3.4 * u - u2v + d["alpha"] * lap(v) - return None, [sp.vstack([fu, fv])] +def _forcing(N): + X, Y = np.meshgrid(np.linspace(0, 1, N), np.linspace(0, 1, N), indexing="ij") + return 5.0 * (((X - 0.3) ** 2 + (Y - 0.6) ** 2) <= 0.04).ravel() -def _brusselator_rows(x, d): - M = d["N"] ** 2 - xs = [x[i] for i in range(2 * M)] - u, v = xs[:M], xs[M:] - a, F = d["alpha"], d["forcing"] - ip, im, jp, jm = d["ip"], d["im"], d["jp"], d["jm"] - fu = [1.0 + u[k] ** 2 * v[k] - 4.4 * u[k] - + a * (u[ip[k]] + u[im[k]] + u[jp[k]] + u[jm[k]] - 4.0 * u[k]) + F[k] - for k in range(M)] - fv = [3.4 * u[k] - u[k] ** 2 * v[k] - + a * (v[ip[k]] + v[im[k]] + v[jp[k]] + v[jm[k]] - 4.0 * v[k]) - for k in range(M)] - return fu, fv +def brusselator(N, forcing=True, interleaved=False): + M, alpha = N * N, 10.0 * (N - 1) ** 2 / 1e4 + F = _forcing(N) if forcing else np.zeros(M) + idx = np.arange(M).reshape(N, N) + nbrs = [np.roll(idx, s, axis=a).ravel() for s in (-1, 1) for a in (0, 1)] -def brusselator_scalar(x, d, ops): - fu, fv = _brusselator_rows(x, d) - return None, fu + fv + def vec(x): + u, v = x[0:M], x[M:2 * M] + def lap(w): + return sum(w[i] for i in nbrs) - 4.0 * w -def brusselator_interleaved(x, d, ops): - # same rows, written per grid cell: (fu_k, fv_k) for each k - fu, fv = _brusselator_rows(x, d) - return None, [t for pair in zip(fu, fv) for t in pair] + u2v = u ** 2 * v + fu = 1.0 + u2v - 4.4 * u + alpha * lap(u) + F.reshape(-1, 1) + fv = 3.4 * u - u2v + alpha * lap(v) + return None, sp.vstack([fu, fv]) + def scalar(x): + u, v = [x[i] for i in range(M)], [x[M + i] for i in range(M)] -def logreg_data(n, seed=0): - rng = np.random.default_rng(seed) - m = 2 * n - A = rng.standard_normal((m, n)) / np.sqrt(n) - y = (rng.random(m) < 0.5).astype(float) - return {"n": n, "m": m, "A": A, "c": A.T @ y, "mu": 1e-2} + def lap(w, k): + return sum(w[i[k]] for i in nbrs) - 4.0 * w[k] + fu = [1.0 + u[k] ** 2 * v[k] - 4.4 * u[k] + alpha * lap(u, k) + F[k] + for k in range(M)] + fv = [3.4 * u[k] - u[k] ** 2 * v[k] + alpha * lap(v, k) + for k in range(M)] + if interleaved: + return None, [t for pair in zip(fu, fv) for t in pair] + return None, fu + fv -def logreg_vec(x, d, ops): - n = d["n"] - f = (ops.sum(ops.softplus(Constant(d["A"]) @ x)) - - Constant(d["c"].reshape(1, -1)) @ x - + 0.5 * d["mu"] * ops.sum(x ** 2)) - return f, [x[0:n - 1] ** 2 + x[1:n] ** 2] + return 2 * M, vec, scalar -def logreg_scalar(x, d, ops): - # the row dot products are matmuls: compact keeps those terms as they are - n, m, A = d["n"], d["m"], d["A"] - xs = [x[i] for i in range(n)] - f = [ops.softplus(Constant(A[i].reshape(1, -1)) @ x) for i in range(m)] - f += [-d["c"][j] * xs[j] + 0.5 * d["mu"] * xs[j] ** 2 for j in range(n)] - return f, [xs[i] ** 2 + xs[i + 1] ** 2 for i in range(n - 1)] - - -def vdp_data(N): - return {"N": N, "n": 3 * N + 2, "h": 10.0 / N} - +def logreg(n): + rng = np.random.default_rng(0) + A = rng.standard_normal((2 * n, n)) / np.sqrt(n) + c = A.T @ (rng.random(2 * n) < 0.5) + mu = 1e-2 -def _vdp_rhs(a, b, u): - return (1.0 - b ** 2) * a - b + u, a + def vec(x): + f = (sp.sum(sp.logistic(Constant(A) @ x)) - Constant(c[None, :]) @ x + + 0.5 * mu * sp.sum(x ** 2)) + return f, x[0:n - 1] ** 2 + x[1:n] ** 2 + def scalar(x): + xs = [x[i] for i in range(n)] + f = [sp.logistic(Constant(a[None, :]) @ x) for a in A] + f += [-c[j] * xs[j] + 0.5 * mu * xs[j] ** 2 for j in range(n)] + return f, [xs[i] ** 2 + xs[i + 1] ** 2 for i in range(n - 1)] -def _vdp_step(a, b, u, h): - k1a, k1b = _vdp_rhs(a, b, u) - k2a, k2b = _vdp_rhs(a + 0.5 * h * k1a, b + 0.5 * h * k1b, u) - k3a, k3b = _vdp_rhs(a + 0.5 * h * k2a, b + 0.5 * h * k2b, u) - k4a, k4b = _vdp_rhs(a + h * k3a, b + h * k3b, u) - return (a + (h / 6.0) * (k1a + 2.0 * k2a + 2.0 * k3a + k4a), - b + (h / 6.0) * (k1b + 2.0 * k2b + 2.0 * k3b + k4b)) + return n, vec, scalar -def vdp_vec(x, d, ops): - N, h = d["N"], d["h"] - x1, x2, u = x[0:N + 1], x[N + 1:2 * N + 2], x[2 * N + 2:3 * N + 2] - f = ops.sum(x1 ** 2) + ops.sum(x2 ** 2) + ops.sum(u ** 2) - na, nb = _vdp_step(x1[0:N], x2[0:N], u, h) - return f, [sp.vstack([x1[1:N + 1] - na, x2[1:N + 1] - nb])] +def vdp(N): + h = 10.0 / N + def rk4(a, b, u): + def rhs(a, b): + return (1.0 - b ** 2) * a - b + u, a -def vdp_scalar(x, d, ops): - # deep terms with heavy sharing: x1[k], x2[k], u[k] and the RK stages - # are reused many times inside each defect - N, h = d["N"], d["h"] - xs = [x[i] for i in range(3 * N + 2)] - x1, x2, u = xs[0:N + 1], xs[N + 1:2 * N + 2], xs[2 * N + 2:] - g1, g2 = [], [] - for k in range(N): - na, nb = _vdp_step(x1[k], x2[k], u[k], h) - g1.append(x1[k + 1] - na) - g2.append(x2[k + 1] - nb) - return [t ** 2 for t in xs], g1 + g2 + k1 = rhs(a, b) + k2 = rhs(a + 0.5 * h * k1[0], b + 0.5 * h * k1[1]) + k3 = rhs(a + 0.5 * h * k2[0], b + 0.5 * h * k2[1]) + k4 = rhs(a + h * k3[0], b + h * k3[1]) + return [s + (h / 6.0) * (p + 2.0 * q + 2.0 * r + t) + for s, p, q, r, t in zip((a, b), k1, k2, k3, k4)] + def vec(x): + x1, x2, u = x[0:N + 1], x[N + 1:2 * N + 2], x[2 * N + 2:3 * N + 2] + na, nb = rk4(x1[0:N], x2[0:N], u) + return (sp.sum(x1 ** 2) + sp.sum(x2 ** 2) + sp.sum(u ** 2), + sp.vstack([x1[1:N + 1] - na, x2[1:N + 1] - nb])) + + def scalar(x): + xs = [x[i] for i in range(3 * N + 2)] + x1, x2, u = xs[0:N + 1], xs[N + 1:2 * N + 2], xs[2 * N + 2:] + steps = [rk4(x1[k], x2[k], u[k]) for k in range(N)] + g = ([x1[k + 1] - s[0] for k, s in enumerate(steps)] + + [x2[k + 1] - s[1] for k, s in enumerate(steps)]) + return [t ** 2 for t in xs], g -OPS = SimpleNamespace(sum=sp.sum, sin=sp.sin, exp=sp.exp, softplus=sp.logistic) + return 3 * N + 2, vec, scalar -# name: (data, vec, scalar, size, objective groups, constraint groups) -PROBLEMS = { - "luksan": (luksan_data, luksan_vec, luksan_scalar, 12, 1, 1), - "brusselator": (brusselator_data, brusselator_vec, brusselator_scalar, - 4, None, 2), - "logreg": (logreg_data, logreg_vec, logreg_scalar, 6, 2 * 6 + 1, 1), - "vdp": (vdp_data, vdp_vec, vdp_scalar, 6, 1, 2), -} +def _n_groups(e): + return len(e.args) if e.opcode == "vstack" else 1 -# ------------------------------------------------------------------ harness +def _build(problem): + n, vec, scalar = problem + f, g = vec(sp.Scope().Variable(n, 1)) + ref = sp.Problem(0.0 if f is None else f, [g]) + f_terms, g_terms = scalar(sp.Scope().Variable(n, 1)) + f = None if f_terms is None else sp.compact(f_terms)[0] + g, perm = sp.compact(g_terms) + p = sp.Problem(0.0 if f is None else sp.sum(f), [g]) + return n, ref, p, f, g, perm -def _n_groups(g): - return len(g.args) if g.opcode == "vstack" else 1 - -def _oracles(f, g, u, sigma, lam): - prob = sp.Problem(0.0 if f is None else f, [g]) - fv = prob.objective_forward(u) - c = np.asarray(prob.constraint_forward(u)).ravel().copy() - grad = np.asarray(prob.gradient()).ravel().copy() - J = prob.jacobian().toarray() - H = prob.hessian(sigma, lam).toarray() - return float(fv), grad, c, J, H - - -def _build(form, d, compact): - """Objective and stacked constraints; perm maps compact rows to terms.""" - x = sp.Scope().Variable(d["n"], 1) - f, g = form(x, d, OPS) - if isinstance(f, list): # scalar objective terms: sum of a stack - f = sp.sum(sp.compact(f)[0] if compact else sp.vstack(f)) - if not compact: - return f, sp.vstack(g) if len(g) > 1 else g[0], None - gc, perm = sp.compact(g) - return f, gc, perm - - -def _point(d, m, seed=1): - rng = np.random.default_rng(seed) - return 0.5 + 0.3 * rng.standard_normal(d["n"]), 1.3, rng.standard_normal(m) - - -def _assert_same(got, expected, perm=None): - rows = slice(None) if perm is None else perm - exp_f, exp_grad, exp_c, exp_J, exp_H = expected - for name, a, b in zip( - ["objective", "gradient", "constraints", "jacobian", "hessian"], - got, (exp_f, exp_grad, exp_c[rows], exp_J[rows], exp_H)): - np.testing.assert_allclose(a, b, err_msg=name, **TOL) - - -# ------------------------------------------------------------------ tests - - -@pytest.mark.parametrize("name", PROBLEMS) -def test_compact_matches_vstack_and_vec(name): - data, vec, scalar, size, f_groups, g_groups = PROBLEMS[name] - d = data(size) - - f_vec, g_vec, _ = _build(vec, d, compact=False) - f_vs, g_vs, _ = _build(scalar, d, compact=False) - f_c, g_c, perm = _build(scalar, d, compact=True) - m = g_vec.shape[0] - assert g_vs.shape == g_c.shape == (m, 1) - if perm is not None: - assert sorted(perm) == list(range(m)) - - u, sigma, lam = _point(d, m) - ref = _oracles(f_vec, g_vec, u, sigma, lam) - # the scalar form is the same math as the vectorized one ... - _assert_same(_oracles(f_vs, g_vs, u, sigma, lam), ref) - # ... and compact keeps it, with multipliers given in compact's row order - lam_c = lam if perm is None else lam[perm] - _assert_same(_oracles(f_c, g_c, u, sigma, lam_c), ref, perm) - - assert _n_groups(g_c) == g_groups - if f_groups is not None: - assert _n_groups(f_c.args[0]) == f_groups - - -@pytest.mark.parametrize("name", PROBLEMS) -def test_compact_at_several_points(name): - # forward/derivative caches must not leak between evaluations - data, vec, scalar, size, _, _ = PROBLEMS[name] - d = data(size) - f_vec, g_vec, _ = _build(vec, d, compact=False) - f_c, g_c, perm = _build(scalar, d, compact=True) - p_vec = sp.Problem(0.0 if f_vec is None else f_vec, [g_vec]) - p_c = sp.Problem(0.0 if f_c is None else f_c, [g_c]) +def _assert_same(n, ref, p, perm): rows = slice(None) if perm is None else perm - for seed in range(4): - u, sigma, lam = _point(d, g_vec.shape[0], seed) - np.testing.assert_allclose(p_c.objective_forward(u), - p_vec.objective_forward(u), **TOL) - np.testing.assert_allclose(p_c.gradient(), p_vec.gradient(), **TOL) - np.testing.assert_allclose( - np.asarray(p_c.constraint_forward(u)).ravel(), - np.asarray(p_vec.constraint_forward(u)).ravel()[rows], **TOL) - np.testing.assert_allclose(p_c.jacobian().toarray(), - p_vec.jacobian().toarray()[rows], **TOL) - np.testing.assert_allclose(p_c.hessian(sigma, lam[rows]).toarray(), - p_vec.hessian(sigma, lam).toarray(), **TOL) - - -def test_brusselator_interleaved_rows_get_permuted(): - d = brusselator_data(3) - M = 9 - x = sp.Scope().Variable(d["n"], 1) - _, terms = brusselator_interleaved(x, d, OPS) - g_c, perm = sp.compact(terms) - assert _n_groups(g_c) == 2 - # all fu rows (even positions) first, then all fv rows - assert list(perm) == list(range(0, 2 * M, 2)) + list(range(1, 2 * M, 2)) - - _, g_vec, _ = _build(brusselator_vec, d, compact=False) - u, sigma, lam = _point(d, 2 * M) - # interleaved row r is vec row (r // 2) + M * (r % 2) - to_vec = np.array([r // 2 + M * (r % 2) for r in range(2 * M)]) - ref = _oracles(None, g_vec, u, sigma, lam) - got = _oracles(None, g_c, u, sigma, lam[to_vec[perm]]) - _assert_same(got, ref, to_vec[perm]) - - -def test_brusselator_varying_forcing_is_a_vector_constant(): - # forcing is 0 on most cells and 5 on some: the merged const must keep - # per-lane values rather than collapse to the first lane's - d = brusselator_data(4) - assert 0 < np.count_nonzero(d["forcing"]) < d["forcing"].size - d0 = dict(d, forcing=np.zeros_like(d["forcing"])) - u, _, _ = _point(d, 2 * 16) - _, g, _ = _build(brusselator_scalar, d, compact=True) - _, g0, _ = _build(brusselator_scalar, d0, compact=True) - c = sp.Problem(0.0, [g]).constraint_forward(u) - c0 = sp.Problem(0.0, [g0]).constraint_forward(u) - np.testing.assert_allclose(np.asarray(c - c0).ravel()[:16], d["forcing"], - **TOL) - - -@pytest.mark.parametrize("N", [1, 2, 20]) -def test_vdp_horizon_lengths(N): - # a one-stage horizon (one lane per group) up to a longer one - d = vdp_data(N) - f_vec, g_vec, _ = _build(vdp_vec, d, compact=False) - f_c, g_c, perm = _build(vdp_scalar, d, compact=True) + for seed in range(3): + rng = np.random.default_rng(seed) + u = 0.5 + 0.3 * rng.standard_normal(n) + lam = rng.standard_normal(ref.total_constraint_size) + for a, b in [ + (p.objective_forward(u), ref.objective_forward(u)), + (p.gradient(), ref.gradient()), + (p.constraint_forward(u), ref.constraint_forward(u)[rows]), + (p.jacobian().toarray(), ref.jacobian().toarray()[rows]), + (p.hessian(1.3, lam[rows]).toarray(), + ref.hessian(1.3, lam).toarray())]: + np.testing.assert_allclose(a, b, **TOL) + + +@pytest.mark.parametrize("make, size, f_groups, g_groups", [ + (luksan, 3, 1, 1), (luksan, 12, 1, 1), (luksan, 50, 1, 1), + (brusselator, 4, None, 2), + (logreg, 6, 13, 1), + (vdp, 1, 1, 2), (vdp, 6, 1, 2), (vdp, 20, 1, 2), +], ids=lambda v: getattr(v, "__name__", str(v))) +def test_compact_matches_vectorized(make, size, f_groups, g_groups): + n, ref, p, f, g, perm = _build(make(size)) assert perm is None - u, sigma, lam = _point(d, 2 * N) - _assert_same(_oracles(f_c, g_c, u, sigma, lam), - _oracles(f_vec, g_vec, u, sigma, lam)) + assert _n_groups(g) == g_groups + assert f_groups is None or _n_groups(f) == f_groups + _assert_same(n, ref, p, perm) -@pytest.mark.parametrize("n", [3, 4, 50]) -def test_luksan_sizes(n): - # n = 3 leaves a single constraint term: nothing to merge it with - d = luksan_data(n) - f_vec, g_vec, _ = _build(luksan_vec, d, compact=False) - f_c, g_c, perm = _build(luksan_scalar, d, compact=True) - assert perm is None - u, sigma, lam = _point(d, n - 2) - _assert_same(_oracles(f_c, g_c, u, sigma, lam), - _oracles(f_vec, g_vec, u, sigma, lam)) +def test_interleaved_rows_are_grouped(): + n, ref, p, f, g, perm = _build(brusselator(3, interleaved=True)) + assert _n_groups(g) == 2 + assert list(perm) == list(range(0, 18, 2)) + list(range(1, 18, 2)) + _assert_same(n, ref, p, None) + + +def test_varying_constant_stays_per_row(): + F = _forcing(4) + assert 0 < np.count_nonzero(F) < F.size + u = np.random.default_rng(0).uniform(0.5, 1.5, 32) + c, c0 = [_build(brusselator(4, forcing=b))[2].constraint_forward(u) + for b in (True, False)] + np.testing.assert_allclose((c - c0)[:16], F, **TOL) def test_compact_shrinks_the_graph(): - # the point of compact: far fewer nodes than one subtree per term - def count(e, seen=None): - seen = set() if seen is None else seen + def count(e, seen): if id(e) in seen: return 0 seen.add(id(e)) - return 1 + sum(count(a, seen) for a in getattr(e, "args", ())) + return 1 + sum(count(a, seen) for a in e.args) - d = luksan_data(200) - _, g_vs, _ = _build(luksan_scalar, d, compact=False) - _, g_c, _ = _build(luksan_scalar, d, compact=True) - assert count(g_c) * 50 < count(g_vs) + n, _, scalar = luksan(200) + terms = scalar(sp.Scope().Variable(n, 1))[1] + assert 50 * count(sp.compact(terms)[0], set()) < count(sp.vstack(terms), set()) From d91fb4609d8122bf7596f1a727f78ab0fead5f5e Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 15:02:27 -0800 Subject: [PATCH 08/16] Tests: check compacted graph size is independent of n, not output shape Replaces _n_groups (which read the vstack structure of compact's result) with count_nodes in conftest: compacting a loop at n and 2n must give the same number of nodes. Co-Authored-By: Claude Opus 5.5 --- tests/conftest.py | 11 ++++ tests/test_vectorize.py | 86 +++++++++++++++++--------------- tests/test_vectorize_problems.py | 37 +++++++------- 3 files changed, 73 insertions(+), 61 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index bd8e38b..1b43a6f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,3 +40,14 @@ def jac(x): scope.set_values(x0) np.testing.assert_allclose(J, J_fd, rtol=rtol, atol=atol, err_msg="Jacobian") np.testing.assert_allclose(H, H_fd, rtol=rtol, atol=atol, err_msg="Hessian") + + +def count_nodes(e): + """Number of distinct nodes in the expression DAG rooted at e.""" + seen, stack = set(), [e] + while stack: + e = stack.pop() + if id(e) not in seen: + seen.add(id(e)) + stack.extend(e.args) + return len(seen) diff --git a/tests/test_vectorize.py b/tests/test_vectorize.py index 1b44b2b..6bc00e8 100644 --- a/tests/test_vectorize.py +++ b/tests/test_vectorize.py @@ -5,32 +5,34 @@ import scipy.sparse as sps import sparsediffpy as sp +from conftest import count_nodes from sparsediffpy.expression import Constant def _evaluate(g, u, lam): - # sum(g) is unchanged by a row permutation, so the objective needs no mapping prob = sp.Problem(sp.sum(g), [g]) f, c = prob.objective_forward(u), prob.constraint_forward(u) return (f, prob.gradient().copy(), c.copy(), prob.jacobian().toarray(), prob.hessian(0.7, lam).toarray()) -def _n_groups(g): - # compact returns one part per group, stacked only if there are several - return len(g.args) if g.opcode == "vstack" else 1 +def assert_compact(build, n, rng): + """build(scope, n) -> scalar terms over n + extra variables. + The compacted graph must not grow with n (the terms were merged), and + value, gradient, Jacobian and Hessian must match vstack(terms), with rows + mapped by perm. + """ + assert count_nodes(sp.compact(build(sp.Scope(), n))[0]) == \ + count_nodes(sp.compact(build(sp.Scope(), 2 * n))[0]) -def assert_compact(build, n_vars, rng, groups): - """build(scope) -> scalar terms. Check the group count, then compare value, - gradient, Jacobian and Hessian with vstack(terms), mapping rows by perm.""" - terms = build(sp.Scope()) - u = rng.uniform(0.5, 1.5, n_vars) + scope = sp.Scope() + terms = build(scope, n) + u = rng.uniform(0.5, 1.5, scope.total_var_size) lam = rng.standard_normal(len(terms)) f, grad, c, J, H = _evaluate(sp.vstack(terms), u, lam) - g, perm = sp.compact(build(sp.Scope())) - assert _n_groups(g) == groups + g, perm = sp.compact(build(sp.Scope(), n)) rows = slice(None) if perm is None else perm got = _evaluate(g, u, lam[rows]) expected = (f, grad, c[rows], J[rows], H) @@ -40,63 +42,65 @@ def assert_compact(build, n_vars, rng, groups): return perm -def test_loop_merges_into_one_vector(rng): - def build(scope): - x = scope.Variable(8, 1) +def test_loop_merges_in_place(rng): + def build(scope, n): + x = scope.Variable(n + 2, 1) return [3 * x[k + 1] ** 3 + 2 * x[k + 2] - x[k] * sp.exp(x[k] - x[k + 1]) - for k in range(6)] - assert assert_compact(build, 8, rng, groups=1) is None # rows in place + for k in range(n)] + assert assert_compact(build, 6, rng) is None def test_varying_constants(rng): - def build(scope): - x = scope.Variable(5, 1) - return [(k + 1.0) * (x[k] - k) ** 2 for k in range(5)] - assert_compact(build, 5, rng, groups=1) + def build(scope, n): + x = scope.Variable(n, 1) + return [(k + 1.0) * (x[k] - k) ** 2 for k in range(n)] + assert_compact(build, 5, rng) def test_interleaved_groups_and_unsupported_terms(rng): A = rng.standard_normal((2, 6)) - def build(scope): - x = scope.Variable(6, 1) + def build(scope, n): + x, z = scope.Variable(n + 1, 1), scope.Variable(6, 1) terms = [] - for k in range(5): + for k in range(n): terms += [sp.sin(x[k]) * x[k + 1], x[k] ** 2 - 2.0 * k] - # row @ x is not merged: kept as is, but still placed via perm - terms += [sp.exp(Constant(A[:1]) @ x), - sp.exp(Constant(sps.csr_matrix(A[1:])) @ x)] + terms += [sp.exp(Constant(A[:1]) @ z), + sp.exp(Constant(sps.csr_matrix(A[1:])) @ z)] return terms - perm = assert_compact(build, 6, rng, groups=4) + perm = assert_compact(build, 5, rng) assert list(perm) == [0, 2, 4, 6, 8, 1, 3, 5, 7, 9, 10, 11] def test_shared_subexpression_and_parameter(rng): - def build(scope): - x = scope.Variable(5, 1) + def build(scope, n): + x = scope.Variable(n + 1, 1) p = scope.Parameter(1, 1, value=2.5) terms = [] - for k in range(4): - d = x[k] - x[k + 1] # shared inside the term + for k in range(n): + d = x[k] - x[k + 1] terms.append(p * d * d + sp.tanh(d)) return terms - assert_compact(build, 5, rng, groups=1) + assert_compact(build, 4, rng) def test_terms_without_args(rng): - def build(scope): + def build(scope, n): y = scope.Variable(1, 1) - return [y, y, sp.exp(y), sp.exp(y)] - assert_compact(build, 1, rng, groups=2) + return [y] * n + [sp.exp(y) for _ in range(n)] + assert_compact(build, 2, rng) def test_different_structure_does_not_merge(rng): - # different variable, exponent or sharing pattern: separate groups - def build(scope): - x, z = scope.Variable(3, 1), scope.Variable(3, 1) - return [x[0] ** 2, x[1] ** 2, z[0] ** 2, x[2] ** 3, - x[0] * x[1], x[1] * x[1]] - assert_compact(build, 6, rng, groups=4) + scope = sp.Scope() + x, z = scope.Variable(3, 1), scope.Variable(3, 1) + terms = [x[0] ** 2, x[1] ** 2, z[0] ** 2, x[2] ** 3, x[0] * x[1], x[1] * x[1]] + g, perm = sp.compact(terms) + assert perm is None + a, b, c, d = rng.uniform(0.5, 1.5, 4) + u = np.array([a, b, c, d, 0.0, 0.0]) + np.testing.assert_allclose(sp.Problem(sp.sum(g), [g]).constraint_forward(u), + [a * a, b * b, d * d, c ** 3, a * b, b * b]) def test_parameter_update_after_compact(): diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py index 2b265ef..b56e981 100644 --- a/tests/test_vectorize_problems.py +++ b/tests/test_vectorize_problems.py @@ -4,6 +4,7 @@ import pytest import sparsediffpy as sp +from conftest import count_nodes from sparsediffpy.expression import Constant TOL = dict(rtol=1e-10, atol=1e-10) @@ -121,10 +122,6 @@ def scalar(x): return 3 * N + 2, vec, scalar -def _n_groups(e): - return len(e.args) if e.opcode == "vstack" else 1 - - def _build(problem): n, vec, scalar = problem f, g = vec(sp.Scope().Variable(n, 1)) @@ -152,23 +149,29 @@ def _assert_same(n, ref, p, perm): np.testing.assert_allclose(a, b, **TOL) -@pytest.mark.parametrize("make, size, f_groups, g_groups", [ - (luksan, 3, 1, 1), (luksan, 12, 1, 1), (luksan, 50, 1, 1), - (brusselator, 4, None, 2), - (logreg, 6, 13, 1), - (vdp, 1, 1, 2), (vdp, 6, 1, 2), (vdp, 20, 1, 2), +@pytest.mark.parametrize("make, size", [ + (luksan, 3), (luksan, 12), (luksan, 50), (brusselator, 4), (logreg, 6), + (vdp, 1), (vdp, 6), (vdp, 20), ], ids=lambda v: getattr(v, "__name__", str(v))) -def test_compact_matches_vectorized(make, size, f_groups, g_groups): +def test_compact_matches_vectorized(make, size): n, ref, p, f, g, perm = _build(make(size)) assert perm is None - assert _n_groups(g) == g_groups - assert f_groups is None or _n_groups(f) == f_groups _assert_same(n, ref, p, perm) +@pytest.mark.parametrize("make, size", [ + (luksan, 10), (brusselator, 3), (logreg, 6), (vdp, 3), +], ids=lambda v: getattr(v, "__name__", str(v))) +def test_graph_size_does_not_grow(make, size): + *_, f1, g1, _ = _build(make(size)) + *_, f2, g2, _ = _build(make(2 * size)) + assert count_nodes(g1) == count_nodes(g2) + if f1 is not None and make is not logreg: + assert count_nodes(f1) == count_nodes(f2) + + def test_interleaved_rows_are_grouped(): n, ref, p, f, g, perm = _build(brusselator(3, interleaved=True)) - assert _n_groups(g) == 2 assert list(perm) == list(range(0, 18, 2)) + list(range(1, 18, 2)) _assert_same(n, ref, p, None) @@ -183,12 +186,6 @@ def test_varying_constant_stays_per_row(): def test_compact_shrinks_the_graph(): - def count(e, seen): - if id(e) in seen: - return 0 - seen.add(id(e)) - return 1 + sum(count(a, seen) for a in e.args) - n, _, scalar = luksan(200) terms = scalar(sp.Scope().Variable(n, 1))[1] - assert 50 * count(sp.compact(terms)[0], set()) < count(sp.vstack(terms), set()) + assert 50 * count_nodes(sp.compact(terms)[0]) < count_nodes(sp.vstack(terms)) From 9e71be1fef5bf2a93f12b6156e85c50977ac5a72 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 15:26:06 -0800 Subject: [PATCH 09/16] compact: keep too-deep terms as is; drop redundant forcing test _lower recurses, so a single very deep term (e.g. a 3000-term sum chain) raised RecursionError; it now falls back to being kept unmerged. The Brusselator forcing test duplicated what test_compact_matches_vectorized already checks. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/vectorize.py | 2 +- tests/test_vectorize.py | 12 ++++++++++++ tests/test_vectorize_problems.py | 19 +++---------------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py index 3bdfee7..ba941f7 100644 --- a/sparsediffpy/vectorize.py +++ b/sparsediffpy/vectorize.py @@ -49,7 +49,7 @@ def compact(terms): raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") try: insts, args = _lower(t, values) - except _Unsupported: + except (_Unsupported, RecursionError): # deep terms are kept as is insts, args = ("single", i), None groups.setdefault(insts, []).append((i, t, args)) diff --git a/tests/test_vectorize.py b/tests/test_vectorize.py index 6bc00e8..08b2a05 100644 --- a/tests/test_vectorize.py +++ b/tests/test_vectorize.py @@ -115,6 +115,18 @@ def test_parameter_update_after_compact(): np.testing.assert_allclose(prob.constraint_forward(u), 5.0 * u) +def test_deep_term_is_kept_as_is(): + x = sp.Scope().Variable(3000, 1) + deep = x[0] + for k in range(1, 3000): + deep = deep + x[k] + g, perm = sp.compact([deep, x[0] ** 2, x[1] ** 2]) + assert perm is None and g.args[0] is deep + u = np.ones(3000) + np.testing.assert_allclose(sp.Problem(sp.sum(g), [g]).constraint_forward(u), + [3000.0, 1.0, 1.0]) + + def test_rejects_non_scalar_and_empty(): x = sp.Scope().Variable(3, 1) with pytest.raises(ValueError): diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py index b56e981..9760273 100644 --- a/tests/test_vectorize_problems.py +++ b/tests/test_vectorize_problems.py @@ -32,14 +32,10 @@ def scalar(x): return n, vec, scalar -def _forcing(N): - X, Y = np.meshgrid(np.linspace(0, 1, N), np.linspace(0, 1, N), indexing="ij") - return 5.0 * (((X - 0.3) ** 2 + (Y - 0.6) ** 2) <= 0.04).ravel() - - -def brusselator(N, forcing=True, interleaved=False): +def brusselator(N, interleaved=False): M, alpha = N * N, 10.0 * (N - 1) ** 2 / 1e4 - F = _forcing(N) if forcing else np.zeros(M) + X, Y = np.meshgrid(np.linspace(0, 1, N), np.linspace(0, 1, N), indexing="ij") + F = 5.0 * (((X - 0.3) ** 2 + (Y - 0.6) ** 2) <= 0.04).ravel() idx = np.arange(M).reshape(N, N) nbrs = [np.roll(idx, s, axis=a).ravel() for s in (-1, 1) for a in (0, 1)] @@ -176,15 +172,6 @@ def test_interleaved_rows_are_grouped(): _assert_same(n, ref, p, None) -def test_varying_constant_stays_per_row(): - F = _forcing(4) - assert 0 < np.count_nonzero(F) < F.size - u = np.random.default_rng(0).uniform(0.5, 1.5, 32) - c, c0 = [_build(brusselator(4, forcing=b))[2].constraint_forward(u) - for b in (True, False)] - np.testing.assert_allclose((c - c0)[:16], F, **TOL) - - def test_compact_shrinks_the_graph(): n, _, scalar = luksan(200) terms = scalar(sp.Scope().Variable(n, 1))[1] From 61e420370ee1843d5927294efe2ae54449dfe347 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 15:26:28 -0800 Subject: [PATCH 10/16] Node: make opcode a required argument Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/expression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sparsediffpy/expression.py b/sparsediffpy/expression.py index bb66e53..c897e02 100644 --- a/sparsediffpy/expression.py +++ b/sparsediffpy/expression.py @@ -1,7 +1,8 @@ """Expression graph: Scope, leaves (Variable/Parameter/Constant), and operators. -Every non-leaf is a generic ``Node(make, args, shape)``: ``make`` receives the -C capsules of ``args`` and returns the node's own capsule. Shapes are always +Every non-leaf is a generic ``Node(make, args, shape, opcode)``: ``make`` +receives the C capsules of ``args`` and returns the node's own capsule, and +``opcode`` names the operation. Shapes are always 2-D ``(d1, d2)`` and values are flattened column-major, like the C engine. """ @@ -90,7 +91,7 @@ class Node(Expression): Compilation only uses ``make``. """ - def __init__(self, make, args, shape, opcode=None, attrs=None): + def __init__(self, make, args, shape, opcode, attrs=None): self.make = make self.args = tuple(args) self.shape = tuple(shape) From 9dd9c0019ec7db10558b74bcfd0bd64e04f00bf3 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:05:02 -0800 Subject: [PATCH 11/16] vectorize: clearer names and numpydoc docstrings Rename insts -> instructions and lanes -> rows, and document compact and its helpers with Parameters/Returns sections, following CVXPY. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/vectorize.py | 133 +++++++++++++++++++++++++++++--------- 1 file changed, 101 insertions(+), 32 deletions(-) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py index ba941f7..11cfbdc 100644 --- a/sparsediffpy/vectorize.py +++ b/sparsediffpy/vectorize.py @@ -34,11 +34,43 @@ class _Unsupported(Exception): def compact(terms): - """Stack scalar expressions like :func:`vstack`, merging identical ones. - - Returns ``(expr, perm)``: ``expr`` has shape ``(len(terms), 1)`` and its - row ``j`` is ``terms[perm[j]]``, or ``perm`` is ``None`` if no row moved. - Terms using other operations (matmul, sum, ...) are kept as they are. + """Stack scalar terms into a column vector, merging terms of the same form. + + ``terms`` is typically the list of scalar objective or constraint terms + built in a Python loop, such as + ``[x[k] ** 2 + x[k + 1] for k in range(n - 1)]``. Terms that apply the + same operations, and differ only in which entries of a Variable they index + or which constants they use, are rebuilt as one vector expression. Terms + using other operations (matmul, sum, ...) are kept as they are. + + Parameters + ---------- + terms : list of Expression or float + The scalar terms to stack; each must have shape ``(1, 1)``. + + Returns + ------- + expr : Expression + An expression of shape ``(len(terms), 1)`` holding one term per row. + Merged terms are grouped together, so rows may be reordered. + perm : numpy.ndarray of int or None + Row ``j`` of ``expr`` is ``terms[perm[j]]``. ``None`` if every term + kept its position, i.e. ``expr`` is ``vstack(terms)``. + + Raises + ------ + ValueError + If ``terms`` is empty or a term does not have shape ``(1, 1)``. + + Examples + -------- + >>> x = sp.Scope().Variable(5, 1) + >>> expr, perm = sp.compact([x[k] ** 2 + x[k + 1] for k in range(4)]) + >>> expr.shape, perm + ((4, 1), None) + >>> expr, perm = sp.compact([x[0] ** 2, sp.exp(x[1]), x[2] ** 2]) + >>> perm + array([0, 2, 1]) """ if not terms: raise ValueError("compact: need at least one term") @@ -48,31 +80,52 @@ def compact(terms): if t.shape != (1, 1): raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") try: - insts, args = _lower(t, values) + instructions, args = _lower(t, values) except (_Unsupported, RecursionError): # deep terms are kept as is - insts, args = ("single", i), None - groups.setdefault(insts, []).append((i, t, args)) + instructions, args = ("single", i), None + groups.setdefault(instructions, []).append((i, t, args)) parts, perm = [], [] - for insts, members in groups.items(): + for instructions, members in groups.items(): perm += [i for i, _, _ in members] parts.append(members[0][1] if len(members) == 1 else - _vectorize(insts, [args for _, _, args in members], values)) + _vectorize(instructions, + [args for _, _, args in members], values)) expr = parts[0] if len(parts) == 1 else atoms.vstack(parts) perm = None if perm == list(range(len(perm))) else np.array(perm) return expr, perm def _lower(term, values): - """Lower a scalar term to ``(insts, args)``. - - ``insts`` lists the term's nodes in post-order as ``(opcode, *operands)``; - operands are value numbers (positions in the list), so a subexpression - shared inside the term is computed once. ``args`` holds the per-term data, - in instruction order: the index of each ``load`` and the value of each - ``const``. + """Lower a scalar term to a list of instructions and its per-term args. + + Two terms with equal ``instructions`` apply the same operations and differ + only in their ``args``, so they can be merged by :func:`_vectorize`. + + Parameters + ---------- + term : Expression + A scalar term of shape ``(1, 1)``. + values : dict + Maps ``id`` to each Variable or Parameter the terms use. Updated in + place with the ones ``term`` uses. + + Returns + ------- + instructions : tuple of tuple + The term's nodes in post-order, each as ``(opcode, *operands)``. + Operands are value numbers (positions in ``instructions``), so a + subexpression shared inside the term is computed once. + args : list + The per-term data, in instruction order: the index of each ``load`` + and the value of each ``const``. + + Raises + ------ + _Unsupported + If ``term`` uses an operation that cannot be vectorized. """ - insts, args, number = [], [], {} # number: id(node) -> value number + instructions, args, number = [], [], {} # number: id(node) -> value number def visit(e): if id(e) in number: @@ -99,31 +152,47 @@ def visit(e): inst = ("power", e.attrs["p"], visit(e.args[0])) else: raise _Unsupported - number[id(e)] = len(insts) - insts.append(inst) + number[id(e)] = len(instructions) + instructions.append(inst) return number[id(e)] visit(term) - return tuple(insts), args + return tuple(instructions), args + + +def _vectorize(instructions, args, values): + """Build one vector expression for a group of terms with equal instructions. + The instructions are run once on column vectors with one row per term: + each ``load`` becomes a gather of the Variable at every term's index, and + each ``const`` a constant vector of every term's value. -def _vectorize(insts, args, values): - """Run ``insts`` once on (lanes, 1) vectors, one lane per term. + Parameters + ---------- + instructions : tuple of tuple + The instructions shared by the terms, as returned by :func:`_lower`. + args : list of list + ``args[i]`` is the args of term ``i``, as returned by :func:`_lower`. + values : dict + Maps ``id`` to each Variable or Parameter the instructions use. - ``args[lane]`` is that term's args; transposed, each ``load``/``const`` - gets one value per lane. + Returns + ------- + Expression + An expression of shape ``(len(args), 1)`` whose row ``i`` is term ``i``. """ - lanes = len(args) - lane_args = iter(zip(*args)) + n_rows = len(args) + arg_columns = iter(zip(*args)) # one load/const arg across all terms vecs = [] # vecs[n]: the vector for value number n - for inst in insts: + for inst in instructions: opcode = inst[0] if opcode == "load": - vecs.append(index_flat(values[inst[1]], next(lane_args), (lanes, 1))) + vecs.append(index_flat(values[inst[1]], next(arg_columns), + (n_rows, 1))) elif opcode == "const": - c = np.array(next(lane_args)) + c = np.array(next(arg_columns)) vecs.append(Constant(c[0] if np.all(c == c[0]) - else c.reshape(lanes, 1))) + else c.reshape(n_rows, 1))) elif opcode == "global": vecs.append(values[inst[1]]) elif opcode == "add": @@ -137,4 +206,4 @@ def _vectorize(insts, args, values): else: vecs.append(getattr(atoms, opcode)(vecs[inst[1]])) # with no load/const (every term is e.g. the scalar y) the result is a scalar - return broadcast(vecs[-1], (lanes, 1)) + return broadcast(vecs[-1], (n_rows, 1)) From 92948e0caad24e11398e9d5bf26d1a9f0911138c Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:20:17 -0800 Subject: [PATCH 12/16] vectorize: rename _Unsupported to UnsupportedTermError Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/vectorize.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py index 11cfbdc..cb11646 100644 --- a/sparsediffpy/vectorize.py +++ b/sparsediffpy/vectorize.py @@ -29,8 +29,8 @@ "normal_cdf", "xexp"} -class _Unsupported(Exception): - pass +class UnsupportedTermError(Exception): + """Raised when a term uses an operation that cannot be vectorized.""" def compact(terms): @@ -81,7 +81,7 @@ def compact(terms): raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") try: instructions, args = _lower(t, values) - except (_Unsupported, RecursionError): # deep terms are kept as is + except (UnsupportedTermError, RecursionError): # deep terms are kept as is instructions, args = ("single", i), None groups.setdefault(instructions, []).append((i, t, args)) @@ -122,7 +122,7 @@ def _lower(term, values): Raises ------ - _Unsupported + UnsupportedTermError If ``term`` uses an operation that cannot be vectorized. """ instructions, args, number = [], [], {} # number: id(node) -> value number @@ -131,7 +131,7 @@ def visit(e): if id(e) in number: return number[id(e)] if e.shape != (1, 1): - raise _Unsupported + raise UnsupportedTermError if isinstance(e, Constant): inst = ("const",) args.append(float(e.value[0, 0])) @@ -139,7 +139,7 @@ def visit(e): values[id(e)] = e inst = ("global", id(e)) elif not isinstance(e, Node): - raise _Unsupported + raise UnsupportedTermError elif e.opcode == "index" and isinstance(e.args[0], Variable): values[id(e.args[0])] = e.args[0] inst = ("load", id(e.args[0])) @@ -151,7 +151,7 @@ def visit(e): elif e.opcode == "power": inst = ("power", e.attrs["p"], visit(e.args[0])) else: - raise _Unsupported + raise UnsupportedTermError number[id(e)] = len(instructions) instructions.append(inst) return number[id(e)] From 6a7c11daec3afd87864912f5f15c559f3ee8b23c Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:26:13 -0800 Subject: [PATCH 13/16] vectorize: call compact's inputs exprs, not terms compact stacks scalar expressions into rows; it does not sum them, so "term" was misleading. Rename UnsupportedTermError to match. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/vectorize.py | 102 +++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py index cb11646..da17876 100644 --- a/sparsediffpy/vectorize.py +++ b/sparsediffpy/vectorize.py @@ -1,8 +1,8 @@ -"""Merge scalar terms written in a Python loop into vector expressions. +"""Merge scalar expressions written in a Python loop into vector expressions. A loop such as ``[x[k] ** 2 + x[k + 1] - 1 for k in range(n - 1)]`` gives the -engine one small expression per term. :func:`compact` lowers each term to a -short list of instructions, in the style of LLVM IR:: +engine one small expression per term. :func:`compact` lowers each expression +to a short list of instructions, in the style of LLVM IR:: %0 = load x ; arg: the index k %1 = power 2.0 %0 @@ -12,7 +12,7 @@ %5 = neg %4 %6 = add %3 %5 -Terms whose instructions are identical differ only in their args, so each such +Expressions whose instructions are identical differ only in their args, so each such group is rebuilt once as a vector expression: a ``load`` becomes a gather of ``x`` at the args, a ``const`` becomes a constant vector. """ @@ -29,38 +29,38 @@ "normal_cdf", "xexp"} -class UnsupportedTermError(Exception): - """Raised when a term uses an operation that cannot be vectorized.""" +class UnsupportedExprError(Exception): + """Raised when an expression uses an operation that cannot be vectorized.""" -def compact(terms): - """Stack scalar terms into a column vector, merging terms of the same form. +def compact(exprs): + """Stack scalar expressions into a column vector, merging those of the same form. - ``terms`` is typically the list of scalar objective or constraint terms + ``exprs`` is typically the list of scalar objective or constraint terms built in a Python loop, such as - ``[x[k] ** 2 + x[k + 1] for k in range(n - 1)]``. Terms that apply the + ``[x[k] ** 2 + x[k + 1] for k in range(n - 1)]``. Expressions that apply the same operations, and differ only in which entries of a Variable they index - or which constants they use, are rebuilt as one vector expression. Terms - using other operations (matmul, sum, ...) are kept as they are. + or which constants they use, are rebuilt as one vector expression. Other + expressions (using matmul, sum, ...) are kept as they are. Parameters ---------- - terms : list of Expression or float - The scalar terms to stack; each must have shape ``(1, 1)``. + exprs : list of Expression or float + The scalar expressions to stack; each must have shape ``(1, 1)``. Returns ------- expr : Expression - An expression of shape ``(len(terms), 1)`` holding one term per row. - Merged terms are grouped together, so rows may be reordered. + An expression of shape ``(len(exprs), 1)`` holding one expression per + row. Merged expressions are grouped together, so rows may be reordered. perm : numpy.ndarray of int or None - Row ``j`` of ``expr`` is ``terms[perm[j]]``. ``None`` if every term - kept its position, i.e. ``expr`` is ``vstack(terms)``. + Row ``j`` of ``expr`` is ``exprs[perm[j]]``. ``None`` if every + expression kept its position, i.e. ``expr`` is ``vstack(exprs)``. Raises ------ ValueError - If ``terms`` is empty or a term does not have shape ``(1, 1)``. + If ``exprs`` is empty or an expression does not have shape ``(1, 1)``. Examples -------- @@ -72,16 +72,17 @@ def compact(terms): >>> perm array([0, 2, 1]) """ - if not terms: - raise ValueError("compact: need at least one term") - values, groups = {}, {} # values: id -> Variable/Parameter the terms use - for i, t in enumerate(terms): + if not exprs: + raise ValueError("compact: need at least one expression") + values, groups = {}, {} # values: id -> Variable/Parameter the exprs use + for i, t in enumerate(exprs): t = cast_to_const(t) if t.shape != (1, 1): - raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") + raise ValueError(f"compact: expression {i} has shape {t.shape}, " + "not (1, 1)") try: instructions, args = _lower(t, values) - except (UnsupportedTermError, RecursionError): # deep terms are kept as is + except (UnsupportedExprError, RecursionError): # deep exprs are kept as is instructions, args = ("single", i), None groups.setdefault(instructions, []).append((i, t, args)) @@ -96,34 +97,34 @@ def compact(terms): return expr, perm -def _lower(term, values): - """Lower a scalar term to a list of instructions and its per-term args. +def _lower(expr, values): + """Lower a scalar expression to a list of instructions and its args. - Two terms with equal ``instructions`` apply the same operations and differ + Two expressions with equal ``instructions`` apply the same operations and differ only in their ``args``, so they can be merged by :func:`_vectorize`. Parameters ---------- - term : Expression - A scalar term of shape ``(1, 1)``. + expr : Expression + A scalar expression of shape ``(1, 1)``. values : dict - Maps ``id`` to each Variable or Parameter the terms use. Updated in - place with the ones ``term`` uses. + Maps ``id`` to each Variable or Parameter the exprs use. Updated in + place with the ones ``expr`` uses. Returns ------- instructions : tuple of tuple - The term's nodes in post-order, each as ``(opcode, *operands)``. + The expression's nodes in post-order, each as ``(opcode, *operands)``. Operands are value numbers (positions in ``instructions``), so a - subexpression shared inside the term is computed once. + subexpression shared inside ``expr`` is computed once. args : list - The per-term data, in instruction order: the index of each ``load`` + The per-expression data, in instruction order: the index of each ``load`` and the value of each ``const``. Raises ------ - UnsupportedTermError - If ``term`` uses an operation that cannot be vectorized. + UnsupportedExprError + If ``expr`` uses an operation that cannot be vectorized. """ instructions, args, number = [], [], {} # number: id(node) -> value number @@ -131,7 +132,7 @@ def visit(e): if id(e) in number: return number[id(e)] if e.shape != (1, 1): - raise UnsupportedTermError + raise UnsupportedExprError if isinstance(e, Constant): inst = ("const",) args.append(float(e.value[0, 0])) @@ -139,7 +140,7 @@ def visit(e): values[id(e)] = e inst = ("global", id(e)) elif not isinstance(e, Node): - raise UnsupportedTermError + raise UnsupportedExprError elif e.opcode == "index" and isinstance(e.args[0], Variable): values[id(e.args[0])] = e.args[0] inst = ("load", id(e.args[0])) @@ -151,38 +152,39 @@ def visit(e): elif e.opcode == "power": inst = ("power", e.attrs["p"], visit(e.args[0])) else: - raise UnsupportedTermError + raise UnsupportedExprError number[id(e)] = len(instructions) instructions.append(inst) return number[id(e)] - visit(term) + visit(expr) return tuple(instructions), args def _vectorize(instructions, args, values): - """Build one vector expression for a group of terms with equal instructions. + """Build one vector expression from expressions with equal instructions. - The instructions are run once on column vectors with one row per term: - each ``load`` becomes a gather of the Variable at every term's index, and - each ``const`` a constant vector of every term's value. + The instructions are run once on column vectors with one row per + expression: each ``load`` becomes a gather of the Variable at every + expression's index, and each ``const`` a constant vector of every + expression's value. Parameters ---------- instructions : tuple of tuple - The instructions shared by the terms, as returned by :func:`_lower`. + The instructions shared by the expressions, as returned by :func:`_lower`. args : list of list - ``args[i]`` is the args of term ``i``, as returned by :func:`_lower`. + ``args[i]`` is the args of expression ``i``, as returned by :func:`_lower`. values : dict Maps ``id`` to each Variable or Parameter the instructions use. Returns ------- Expression - An expression of shape ``(len(args), 1)`` whose row ``i`` is term ``i``. + An expression of shape ``(len(args), 1)``; row ``i`` is expression ``i``. """ n_rows = len(args) - arg_columns = iter(zip(*args)) # one load/const arg across all terms + arg_columns = iter(zip(*args)) # one load/const arg across all expressions vecs = [] # vecs[n]: the vector for value number n for inst in instructions: opcode = inst[0] @@ -205,5 +207,5 @@ def _vectorize(instructions, args, values): vecs.append(vecs[inst[2]] ** inst[1]) else: vecs.append(getattr(atoms, opcode)(vecs[inst[1]])) - # with no load/const (every term is e.g. the scalar y) the result is a scalar + # with no load/const (every expression is e.g. y) the result is a scalar return broadcast(vecs[-1], (n_rows, 1)) From db62256d425479ebd6d6c8150f7382e77d5dd2d3 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:34:02 -0800 Subject: [PATCH 14/16] Problem: always compact runs of scalar constraints Like an LLVM pass pipeline, Problem now runs compact on each run of consecutive scalar constraints. Merged rows are gathered back into the given order with one index node, so constraint values, Jacobian rows and Lagrange multipliers keep their order. test_vectorize_problems now builds the scalar problems directly with Problem. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/problem.py | 37 ++++++++++++++++++++++++- tests/test_vectorize_problems.py | 46 ++++++++++++++++---------------- 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/sparsediffpy/problem.py b/sparsediffpy/problem.py index 913e489..a5e24e3 100644 --- a/sparsediffpy/problem.py +++ b/sparsediffpy/problem.py @@ -10,7 +10,34 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy.expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const +from sparsediffpy.expression import ( + Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const, + index_flat, +) +from sparsediffpy.vectorize import compact + + +def _compact_in_order(exprs): + """``compact(exprs)`` with its rows gathered back into their given order.""" + expr, perm = compact(exprs) + if perm is None: + return expr + return index_flat(expr, np.argsort(perm), expr.shape) + + +def _compact_constraints(constraints): + """Compact each run of consecutive scalar constraints, keeping row order.""" + out, run = [], [] + for c in constraints + [None]: + if c is not None and c.shape == (1, 1): + run.append(c) + continue + if run: + out.append(_compact_in_order(run)) + run = [] + if c is not None: + out.append(c) + return out def _find_scope(exprs): @@ -99,6 +126,12 @@ class Problem: gradient/jacobian are skipped when the matching forward has not moved. Skipped calls return the cached object, which the problem owns: do not modify it in place. + + Each run of consecutive scalar constraints, such as one per iteration of + a Python loop, is merged into vector expressions with + :func:`~sparsediffpy.compact`. Constraint rows keep the order they were + given in. ``objective`` and ``constraints`` hold the expressions as built + in the engine. """ def __init__(self, objective, constraints=(), verbose=False): @@ -106,6 +139,8 @@ def __init__(self, objective, constraints=(), verbose=False): constraints = [cast_to_const(c) for c in constraints] if objective.shape != (1, 1): raise ValueError(f"Objective must be scalar, got {objective.shape}") + constraints = _compact_constraints(constraints) + self.objective, self.constraints = objective, constraints self._scope = _find_scope([objective, *constraints]) build = _Builder(self._scope) obj_cap = build(objective) diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py index 9760273..d564a09 100644 --- a/tests/test_vectorize_problems.py +++ b/tests/test_vectorize_problems.py @@ -1,4 +1,4 @@ -"""compact() on benchmark NLPs must match the hand-vectorized formulation.""" +"""Benchmark NLPs written as scalar loops must match their vectorized form.""" import numpy as np import pytest @@ -119,18 +119,17 @@ def scalar(x): def _build(problem): + """The hand-vectorized problem and the one written as scalar loops.""" n, vec, scalar = problem f, g = vec(sp.Scope().Variable(n, 1)) ref = sp.Problem(0.0 if f is None else f, [g]) f_terms, g_terms = scalar(sp.Scope().Variable(n, 1)) - f = None if f_terms is None else sp.compact(f_terms)[0] - g, perm = sp.compact(g_terms) - p = sp.Problem(0.0 if f is None else sp.sum(f), [g]) - return n, ref, p, f, g, perm + p = sp.Problem(0.0 if f_terms is None else sum(f_terms), g_terms) + return n, ref, p -def _assert_same(n, ref, p, perm): - rows = slice(None) if perm is None else perm +def _assert_same(n, ref, p, rows=slice(None)): + """p matches ref, whose constraint rows are ``rows`` of p's.""" for seed in range(3): rng = np.random.default_rng(seed) u = 0.5 + 0.3 * rng.standard_normal(n) @@ -145,34 +144,35 @@ def _assert_same(n, ref, p, perm): np.testing.assert_allclose(a, b, **TOL) +def _constraint_sizes(p): + return [count_nodes(c) for c in p.constraints] + + @pytest.mark.parametrize("make, size", [ (luksan, 3), (luksan, 12), (luksan, 50), (brusselator, 4), (logreg, 6), (vdp, 1), (vdp, 6), (vdp, 20), ], ids=lambda v: getattr(v, "__name__", str(v))) -def test_compact_matches_vectorized(make, size): - n, ref, p, f, g, perm = _build(make(size)) - assert perm is None - _assert_same(n, ref, p, perm) +def test_scalar_problem_matches_vectorized(make, size): + _assert_same(*_build(make(size))) @pytest.mark.parametrize("make, size", [ (luksan, 10), (brusselator, 3), (logreg, 6), (vdp, 3), ], ids=lambda v: getattr(v, "__name__", str(v))) def test_graph_size_does_not_grow(make, size): - *_, f1, g1, _ = _build(make(size)) - *_, f2, g2, _ = _build(make(2 * size)) - assert count_nodes(g1) == count_nodes(g2) - if f1 is not None and make is not logreg: - assert count_nodes(f1) == count_nodes(f2) + p1, p2 = _build(make(size))[2], _build(make(2 * size))[2] + assert _constraint_sizes(p1) == _constraint_sizes(p2) -def test_interleaved_rows_are_grouped(): - n, ref, p, f, g, perm = _build(brusselator(3, interleaved=True)) - assert list(perm) == list(range(0, 18, 2)) + list(range(1, 18, 2)) - _assert_same(n, ref, p, None) +def test_interleaved_rows_keep_their_order(): + n, ref, p = _build(brusselator(3, interleaved=True)) + assert len(p.constraints) == 1 + M = 9 # fu_k, fv_k alternate; ref stacks all fu, then all fv + _assert_same(n, ref, p, np.arange(2 * M).reshape(2, M).T.ravel()) -def test_compact_shrinks_the_graph(): +def test_problem_shrinks_the_graph(): n, _, scalar = luksan(200) - terms = scalar(sp.Scope().Variable(n, 1))[1] - assert 50 * count_nodes(sp.compact(terms)[0]) < count_nodes(sp.vstack(terms)) + g_terms = scalar(sp.Scope().Variable(n, 1))[1] + p = sp.Problem(0.0, g_terms) + assert 50 * sum(_constraint_sizes(p)) < count_nodes(sp.vstack(g_terms)) From ce94883d0596be16db7516c9f7183583de3ede0c Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:45:38 -0800 Subject: [PATCH 15/16] Tests: call compact's inputs exprs, matching the docstrings Co-Authored-By: Claude Opus 5.5 --- tests/test_vectorize.py | 38 ++++++++++++++++---------------- tests/test_vectorize_problems.py | 10 ++++----- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_vectorize.py b/tests/test_vectorize.py index 08b2a05..4d9f856 100644 --- a/tests/test_vectorize.py +++ b/tests/test_vectorize.py @@ -1,4 +1,4 @@ -"""compact(terms) must merge what it can and match vstack(terms) exactly.""" +"""compact(exprs) must merge what it can and match vstack(exprs) exactly.""" import numpy as np import pytest @@ -17,20 +17,20 @@ def _evaluate(g, u, lam): def assert_compact(build, n, rng): - """build(scope, n) -> scalar terms over n + extra variables. + """build(scope, n) -> scalar expressions over n + extra variables. - The compacted graph must not grow with n (the terms were merged), and - value, gradient, Jacobian and Hessian must match vstack(terms), with rows + The compacted graph must not grow with n (the expressions were merged), and + value, gradient, Jacobian and Hessian must match vstack(exprs), with rows mapped by perm. """ assert count_nodes(sp.compact(build(sp.Scope(), n))[0]) == \ count_nodes(sp.compact(build(sp.Scope(), 2 * n))[0]) scope = sp.Scope() - terms = build(scope, n) + exprs = build(scope, n) u = rng.uniform(0.5, 1.5, scope.total_var_size) - lam = rng.standard_normal(len(terms)) - f, grad, c, J, H = _evaluate(sp.vstack(terms), u, lam) + lam = rng.standard_normal(len(exprs)) + f, grad, c, J, H = _evaluate(sp.vstack(exprs), u, lam) g, perm = sp.compact(build(sp.Scope(), n)) rows = slice(None) if perm is None else perm @@ -57,17 +57,17 @@ def build(scope, n): assert_compact(build, 5, rng) -def test_interleaved_groups_and_unsupported_terms(rng): +def test_interleaved_groups_and_unsupported_exprs(rng): A = rng.standard_normal((2, 6)) def build(scope, n): x, z = scope.Variable(n + 1, 1), scope.Variable(6, 1) - terms = [] + exprs = [] for k in range(n): - terms += [sp.sin(x[k]) * x[k + 1], x[k] ** 2 - 2.0 * k] - terms += [sp.exp(Constant(A[:1]) @ z), + exprs += [sp.sin(x[k]) * x[k + 1], x[k] ** 2 - 2.0 * k] + exprs += [sp.exp(Constant(A[:1]) @ z), sp.exp(Constant(sps.csr_matrix(A[1:])) @ z)] - return terms + return exprs perm = assert_compact(build, 5, rng) assert list(perm) == [0, 2, 4, 6, 8, 1, 3, 5, 7, 9, 10, 11] @@ -76,15 +76,15 @@ def test_shared_subexpression_and_parameter(rng): def build(scope, n): x = scope.Variable(n + 1, 1) p = scope.Parameter(1, 1, value=2.5) - terms = [] + exprs = [] for k in range(n): d = x[k] - x[k + 1] - terms.append(p * d * d + sp.tanh(d)) - return terms + exprs.append(p * d * d + sp.tanh(d)) + return exprs assert_compact(build, 4, rng) -def test_terms_without_args(rng): +def test_exprs_without_args(rng): def build(scope, n): y = scope.Variable(1, 1) return [y] * n + [sp.exp(y) for _ in range(n)] @@ -94,8 +94,8 @@ def build(scope, n): def test_different_structure_does_not_merge(rng): scope = sp.Scope() x, z = scope.Variable(3, 1), scope.Variable(3, 1) - terms = [x[0] ** 2, x[1] ** 2, z[0] ** 2, x[2] ** 3, x[0] * x[1], x[1] * x[1]] - g, perm = sp.compact(terms) + exprs = [x[0] ** 2, x[1] ** 2, z[0] ** 2, x[2] ** 3, x[0] * x[1], x[1] * x[1]] + g, perm = sp.compact(exprs) assert perm is None a, b, c, d = rng.uniform(0.5, 1.5, 4) u = np.array([a, b, c, d, 0.0, 0.0]) @@ -115,7 +115,7 @@ def test_parameter_update_after_compact(): np.testing.assert_allclose(prob.constraint_forward(u), 5.0 * u) -def test_deep_term_is_kept_as_is(): +def test_deep_expr_is_kept_as_is(): x = sp.Scope().Variable(3000, 1) deep = x[0] for k in range(1, 3000): diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py index d564a09..e36960a 100644 --- a/tests/test_vectorize_problems.py +++ b/tests/test_vectorize_problems.py @@ -123,8 +123,8 @@ def _build(problem): n, vec, scalar = problem f, g = vec(sp.Scope().Variable(n, 1)) ref = sp.Problem(0.0 if f is None else f, [g]) - f_terms, g_terms = scalar(sp.Scope().Variable(n, 1)) - p = sp.Problem(0.0 if f_terms is None else sum(f_terms), g_terms) + f_exprs, g_exprs = scalar(sp.Scope().Variable(n, 1)) + p = sp.Problem(0.0 if f_exprs is None else sum(f_exprs), g_exprs) return n, ref, p @@ -173,6 +173,6 @@ def test_interleaved_rows_keep_their_order(): def test_problem_shrinks_the_graph(): n, _, scalar = luksan(200) - g_terms = scalar(sp.Scope().Variable(n, 1))[1] - p = sp.Problem(0.0, g_terms) - assert 50 * sum(_constraint_sizes(p)) < count_nodes(sp.vstack(g_terms)) + g_exprs = scalar(sp.Scope().Variable(n, 1))[1] + p = sp.Problem(0.0, g_exprs) + assert 50 * sum(_constraint_sizes(p)) < count_nodes(sp.vstack(g_exprs)) From b49ba0582e6e62941100afd8d839fe40d0fb85df Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:46:42 -0800 Subject: [PATCH 16/16] Tests: keep count_calls docstring within 85 columns Co-Authored-By: Claude Opus 5.5 --- tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index 00e879a..7355665 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -278,7 +278,7 @@ def test_unset_parameter_raises(scope): # --- Caching ------------------------------------------------------------------ def count_calls(monkeypatch, name): - """Count calls to the engine function `name` as seen from sparsediffpy.problem.""" + """Count calls to engine function `name` as seen from sparsediffpy.problem.""" calls, real = [], getattr(problem._C, name) def counted(*args):