From e842e7be872c23ad1bcccf9a11a6155f0b0bba1c Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 15:34:35 -0800 Subject: [PATCH 1/7] compact: reduce unmerged sums horizontally, like LLVM's SLP reductions A term that merges with no other term may be a long chain of adds, e.g. an objective accumulated as f = f + term_k in a loop. Its operands are flattened (iteratively, so deep chains work), compacted and summed, and the result is kept only if the graph gets smaller. 20000-term loop objective: Problem build 32 s -> 5 ms, objective + gradient + Hessian 10.5 s -> 4 ms, same value. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/vectorize.py | 38 +++++++++++++++++++++++++++++++++++--- tests/test_vectorize.py | 24 ++++++++++++++++++++---- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py index ba941f7..48ff0a9 100644 --- a/sparsediffpy/vectorize.py +++ b/sparsediffpy/vectorize.py @@ -15,6 +15,12 @@ Terms whose instructions are identical differ only in their args, so each such group is rebuilt once as a vector expression: a ``load`` becomes a gather of ``x`` at the args, a ``const`` becomes a constant vector. + +A term that merges with no other term may itself be a long sum, such as +``t = t + x[k]`` accumulated in a loop. Like LLVM's horizontal reduction, its +chain of adds is flattened into operands, which are compacted and summed: +``sum(compact(operands))``. This is kept only if the graph gets smaller, and +it reassociates the sum (results can differ in the last bits). """ import numpy as np @@ -38,7 +44,8 @@ def compact(terms): Returns ``(expr, perm)``: ``expr`` has shape ``(len(terms), 1)`` and its row ``j`` is ``terms[perm[j]]``, or ``perm`` is ``None`` if no row moved. - Terms using other operations (matmul, sum, ...) are kept as they are. + Terms using other operations (matmul, sum, ...) are kept as they are, + unless they are a sum of mergeable operands. """ if not terms: raise ValueError("compact: need at least one term") @@ -49,20 +56,45 @@ def compact(terms): raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") try: insts, args = _lower(t, values) - except (_Unsupported, RecursionError): # deep terms are kept as is + except (_Unsupported, RecursionError): insts, args = ("single", i), None groups.setdefault(insts, []).append((i, t, args)) parts, perm = [], [] for insts, members in groups.items(): perm += [i for i, _, _ in members] - parts.append(members[0][1] if len(members) == 1 else + parts.append(_reduce(members[0][1]) if len(members) == 1 else _vectorize(insts, [args for _, _, args in members], values)) expr = parts[0] if len(parts) == 1 else atoms.vstack(parts) perm = None if perm == list(range(len(perm))) else np.array(perm) return expr, perm +def _reduce(term): + """``sum(compact(operands))`` for a chain of adds, if that is smaller.""" + operands, stack = [], [term] + while stack: + e = stack.pop() + if isinstance(e, Node) and e.opcode == "add": + stack.extend(reversed(e.args)) + else: + operands.append(e) + if len(operands) < 2: + return term + reduced = atoms.sum(compact(operands)[0]) + return reduced if _count_nodes(reduced) < _count_nodes(term) else term + + +def _count_nodes(e): + seen, stack = set(), [e] + while stack: + e = stack.pop() + if id(e) not in seen: + seen.add(id(e)) + stack.extend(e.args) + return len(seen) + + def _lower(term, values): """Lower a scalar term to ``(insts, args)``. diff --git a/tests/test_vectorize.py b/tests/test_vectorize.py index 08b2a05..22cb72d 100644 --- a/tests/test_vectorize.py +++ b/tests/test_vectorize.py @@ -115,16 +115,32 @@ def test_parameter_update_after_compact(): np.testing.assert_allclose(prob.constraint_forward(u), 5.0 * u) -def test_deep_term_is_kept_as_is(): +def test_long_sum_is_reduced(): x = sp.Scope().Variable(3000, 1) deep = x[0] for k in range(1, 3000): deep = deep + x[k] g, perm = sp.compact([deep, x[0] ** 2, x[1] ** 2]) - assert perm is None and g.args[0] is deep - u = np.ones(3000) + assert perm is None and count_nodes(g.args[0]) == 3 + u = np.arange(3000.0) np.testing.assert_allclose(sp.Problem(sp.sum(g), [g]).constraint_forward(u), - [3000.0, 1.0, 1.0]) + [u.sum(), 0.0, 1.0]) + + +def test_accumulated_objective_matches(rng): + def build(scope, n): + x = scope.Variable(n + 1, 1) + f = 0.0 + for k in range(n): + f = f + 100.0 * (x[k] ** 2 - x[k + 1]) ** 2 - (k + 1.0) * x[k] + return [f] + assert_compact(build, 8, rng) + + +def test_unprofitable_sum_is_kept(): + x = sp.Scope().Variable(2, 1) + t = x[0] ** 2 + sp.exp(x[1]) + assert sp.compact([t])[0] is t def test_rejects_non_scalar_and_empty(): From a4b5470e71610d9684cdfc013ee8da12c4a67ab8 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:05:02 -0800 Subject: [PATCH 2/7] vectorize: clearer names and numpydoc docstrings Rename insts -> instructions and lanes -> rows, and document compact and its helpers with Parameters/Returns sections, following CVXPY. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/vectorize.py | 159 ++++++++++++++++++++++++++++++-------- 1 file changed, 127 insertions(+), 32 deletions(-) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py index 48ff0a9..443751b 100644 --- a/sparsediffpy/vectorize.py +++ b/sparsediffpy/vectorize.py @@ -40,12 +40,44 @@ class _Unsupported(Exception): def compact(terms): - """Stack scalar expressions like :func:`vstack`, merging identical ones. + """Stack scalar terms into a column vector, merging terms of the same form. - Returns ``(expr, perm)``: ``expr`` has shape ``(len(terms), 1)`` and its - row ``j`` is ``terms[perm[j]]``, or ``perm`` is ``None`` if no row moved. - Terms using other operations (matmul, sum, ...) are kept as they are, - unless they are a sum of mergeable operands. + ``terms`` is typically the list of scalar objective or constraint terms + built in a Python loop, such as + ``[x[k] ** 2 + x[k + 1] for k in range(n - 1)]``. Terms that apply the + same operations, and differ only in which entries of a Variable they index + or which constants they use, are rebuilt as one vector expression. Terms + using other operations (matmul, sum, ...) are kept as they are, unless + they are a sum of mergeable operands. + + Parameters + ---------- + terms : list of Expression or float + The scalar terms to stack; each must have shape ``(1, 1)``. + + Returns + ------- + expr : Expression + An expression of shape ``(len(terms), 1)`` holding one term per row. + Merged terms are grouped together, so rows may be reordered. + perm : numpy.ndarray of int or None + Row ``j`` of ``expr`` is ``terms[perm[j]]``. ``None`` if every term + kept its position, i.e. ``expr`` is ``vstack(terms)``. + + Raises + ------ + ValueError + If ``terms`` is empty or a term does not have shape ``(1, 1)``. + + Examples + -------- + >>> x = sp.Scope().Variable(5, 1) + >>> expr, perm = sp.compact([x[k] ** 2 + x[k + 1] for k in range(4)]) + >>> expr.shape, perm + ((4, 1), None) + >>> expr, perm = sp.compact([x[0] ** 2, sp.exp(x[1]), x[2] ** 2]) + >>> perm + array([0, 2, 1]) """ if not terms: raise ValueError("compact: need at least one term") @@ -55,23 +87,38 @@ def compact(terms): if t.shape != (1, 1): raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") try: - insts, args = _lower(t, values) + instructions, args = _lower(t, values) except (_Unsupported, RecursionError): - insts, args = ("single", i), None - groups.setdefault(insts, []).append((i, t, args)) + instructions, args = ("single", i), None + groups.setdefault(instructions, []).append((i, t, args)) parts, perm = [], [] - for insts, members in groups.items(): + for instructions, members in groups.items(): perm += [i for i, _, _ in members] parts.append(_reduce(members[0][1]) if len(members) == 1 else - _vectorize(insts, [args for _, _, args in members], values)) + _vectorize(instructions, [args for _, _, args in members], + values)) expr = parts[0] if len(parts) == 1 else atoms.vstack(parts) perm = None if perm == list(range(len(perm))) else np.array(perm) return expr, perm def _reduce(term): - """``sum(compact(operands))`` for a chain of adds, if that is smaller.""" + """Rewrite a sum of many operands as ``sum(compact(operands))``. + + The chain of ``add`` nodes in ``term`` is flattened into its operands, + which are compacted and summed. This reassociates the sum. + + Parameters + ---------- + term : Expression + A scalar term that merged with no other term. + + Returns + ------- + Expression + The rewritten sum if it has fewer nodes than ``term``, else ``term``. + """ operands, stack = [], [term] while stack: e = stack.pop() @@ -86,6 +133,18 @@ def _reduce(term): def _count_nodes(e): + """Count the distinct nodes in the DAG rooted at ``e``. + + Parameters + ---------- + e : Expression + The root of the DAG. + + Returns + ------- + int + The number of distinct nodes, counting shared subexpressions once. + """ seen, stack = set(), [e] while stack: e = stack.pop() @@ -96,15 +155,35 @@ def _count_nodes(e): def _lower(term, values): - """Lower a scalar term to ``(insts, args)``. + """Lower a scalar term to a list of instructions and its per-term args. + + Two terms with equal ``instructions`` apply the same operations and differ + only in their ``args``, so they can be merged by :func:`_vectorize`. - ``insts`` lists the term's nodes in post-order as ``(opcode, *operands)``; - operands are value numbers (positions in the list), so a subexpression - shared inside the term is computed once. ``args`` holds the per-term data, - in instruction order: the index of each ``load`` and the value of each - ``const``. + Parameters + ---------- + term : Expression + A scalar term of shape ``(1, 1)``. + values : dict + Maps ``id`` to each Variable or Parameter the terms use. Updated in + place with the ones ``term`` uses. + + Returns + ------- + instructions : tuple of tuple + The term's nodes in post-order, each as ``(opcode, *operands)``. + Operands are value numbers (positions in ``instructions``), so a + subexpression shared inside the term is computed once. + args : list + The per-term data, in instruction order: the index of each ``load`` + and the value of each ``const``. + + Raises + ------ + _Unsupported + If ``term`` uses an operation that cannot be vectorized. """ - insts, args, number = [], [], {} # number: id(node) -> value number + instructions, args, number = [], [], {} # number: id(node) -> value number def visit(e): if id(e) in number: @@ -131,31 +210,47 @@ def visit(e): inst = ("power", e.attrs["p"], visit(e.args[0])) else: raise _Unsupported - number[id(e)] = len(insts) - insts.append(inst) + number[id(e)] = len(instructions) + instructions.append(inst) return number[id(e)] visit(term) - return tuple(insts), args + return tuple(instructions), args + + +def _vectorize(instructions, args, values): + """Build one vector expression for a group of terms with equal instructions. + The instructions are run once on column vectors with one row per term: + each ``load`` becomes a gather of the Variable at every term's index, and + each ``const`` a constant vector of every term's value. -def _vectorize(insts, args, values): - """Run ``insts`` once on (lanes, 1) vectors, one lane per term. + Parameters + ---------- + instructions : tuple of tuple + The instructions shared by the terms, as returned by :func:`_lower`. + args : list of list + ``args[i]`` is the args of term ``i``, as returned by :func:`_lower`. + values : dict + Maps ``id`` to each Variable or Parameter the instructions use. - ``args[lane]`` is that term's args; transposed, each ``load``/``const`` - gets one value per lane. + Returns + ------- + Expression + An expression of shape ``(len(args), 1)`` whose row ``i`` is term ``i``. """ - lanes = len(args) - lane_args = iter(zip(*args)) + n_rows = len(args) + arg_columns = iter(zip(*args)) # one load/const arg across all terms vecs = [] # vecs[n]: the vector for value number n - for inst in insts: + for inst in instructions: opcode = inst[0] if opcode == "load": - vecs.append(index_flat(values[inst[1]], next(lane_args), (lanes, 1))) + vecs.append(index_flat(values[inst[1]], next(arg_columns), + (n_rows, 1))) elif opcode == "const": - c = np.array(next(lane_args)) + c = np.array(next(arg_columns)) vecs.append(Constant(c[0] if np.all(c == c[0]) - else c.reshape(lanes, 1))) + else c.reshape(n_rows, 1))) elif opcode == "global": vecs.append(values[inst[1]]) elif opcode == "add": @@ -169,4 +264,4 @@ def _vectorize(insts, args, values): else: vecs.append(getattr(atoms, opcode)(vecs[inst[1]])) # with no load/const (every term is e.g. the scalar y) the result is a scalar - return broadcast(vecs[-1], (lanes, 1)) + return broadcast(vecs[-1], (n_rows, 1)) From f031b4fc33c22aabd78fc69b833523e04bcf7758 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:20:17 -0800 Subject: [PATCH 3/7] vectorize: rename _Unsupported to UnsupportedTermError Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/vectorize.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py index 443751b..a05623a 100644 --- a/sparsediffpy/vectorize.py +++ b/sparsediffpy/vectorize.py @@ -35,8 +35,8 @@ "normal_cdf", "xexp"} -class _Unsupported(Exception): - pass +class UnsupportedTermError(Exception): + """Raised when a term uses an operation that cannot be vectorized.""" def compact(terms): @@ -88,7 +88,7 @@ def compact(terms): raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") try: instructions, args = _lower(t, values) - except (_Unsupported, RecursionError): + except (UnsupportedTermError, RecursionError): instructions, args = ("single", i), None groups.setdefault(instructions, []).append((i, t, args)) @@ -180,7 +180,7 @@ def _lower(term, values): Raises ------ - _Unsupported + UnsupportedTermError If ``term`` uses an operation that cannot be vectorized. """ instructions, args, number = [], [], {} # number: id(node) -> value number @@ -189,7 +189,7 @@ def visit(e): if id(e) in number: return number[id(e)] if e.shape != (1, 1): - raise _Unsupported + raise UnsupportedTermError if isinstance(e, Constant): inst = ("const",) args.append(float(e.value[0, 0])) @@ -197,7 +197,7 @@ def visit(e): values[id(e)] = e inst = ("global", id(e)) elif not isinstance(e, Node): - raise _Unsupported + raise UnsupportedTermError elif e.opcode == "index" and isinstance(e.args[0], Variable): values[id(e.args[0])] = e.args[0] inst = ("load", id(e.args[0])) @@ -209,7 +209,7 @@ def visit(e): elif e.opcode == "power": inst = ("power", e.attrs["p"], visit(e.args[0])) else: - raise _Unsupported + raise UnsupportedTermError number[id(e)] = len(instructions) instructions.append(inst) return number[id(e)] From 9cfd969284c7667379a8da5dfb686353f6e71d7e Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:26:00 -0800 Subject: [PATCH 4/7] vectorize: call compact's inputs exprs, not terms compact stacks scalar expressions into rows; it does not sum them, so "term" was misleading. Rename UnsupportedTermError to match. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/vectorize.py | 120 +++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 59 deletions(-) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py index a05623a..48bf9a6 100644 --- a/sparsediffpy/vectorize.py +++ b/sparsediffpy/vectorize.py @@ -1,8 +1,8 @@ -"""Merge scalar terms written in a Python loop into vector expressions. +"""Merge scalar expressions written in a Python loop into vector expressions. A loop such as ``[x[k] ** 2 + x[k + 1] - 1 for k in range(n - 1)]`` gives the -engine one small expression per term. :func:`compact` lowers each term to a -short list of instructions, in the style of LLVM IR:: +engine one small expression per term. :func:`compact` lowers each expression +to a short list of instructions, in the style of LLVM IR:: %0 = load x ; arg: the index k %1 = power 2.0 %0 @@ -12,11 +12,11 @@ %5 = neg %4 %6 = add %3 %5 -Terms whose instructions are identical differ only in their args, so each such +Expressions whose instructions are identical differ only in their args, so each such group is rebuilt once as a vector expression: a ``load`` becomes a gather of ``x`` at the args, a ``const`` becomes a constant vector. -A term that merges with no other term may itself be a long sum, such as +An expression that merges with no other may itself be a long sum, such as ``t = t + x[k]`` accumulated in a loop. Like LLVM's horizontal reduction, its chain of adds is flattened into operands, which are compacted and summed: ``sum(compact(operands))``. This is kept only if the graph gets smaller, and @@ -35,39 +35,39 @@ "normal_cdf", "xexp"} -class UnsupportedTermError(Exception): - """Raised when a term uses an operation that cannot be vectorized.""" +class UnsupportedExprError(Exception): + """Raised when an expression uses an operation that cannot be vectorized.""" -def compact(terms): - """Stack scalar terms into a column vector, merging terms of the same form. +def compact(exprs): + """Stack scalar expressions into a column vector, merging those of the same form. - ``terms`` is typically the list of scalar objective or constraint terms + ``exprs`` is typically the list of scalar objective or constraint terms built in a Python loop, such as - ``[x[k] ** 2 + x[k + 1] for k in range(n - 1)]``. Terms that apply the + ``[x[k] ** 2 + x[k + 1] for k in range(n - 1)]``. Expressions that apply the same operations, and differ only in which entries of a Variable they index - or which constants they use, are rebuilt as one vector expression. Terms - using other operations (matmul, sum, ...) are kept as they are, unless + or which constants they use, are rebuilt as one vector expression. Other + expressions (using matmul, sum, ...) are kept as they are, unless they are a sum of mergeable operands. Parameters ---------- - terms : list of Expression or float - The scalar terms to stack; each must have shape ``(1, 1)``. + exprs : list of Expression or float + The scalar expressions to stack; each must have shape ``(1, 1)``. Returns ------- expr : Expression - An expression of shape ``(len(terms), 1)`` holding one term per row. - Merged terms are grouped together, so rows may be reordered. + An expression of shape ``(len(exprs), 1)`` holding one expression per + row. Merged expressions are grouped together, so rows may be reordered. perm : numpy.ndarray of int or None - Row ``j`` of ``expr`` is ``terms[perm[j]]``. ``None`` if every term - kept its position, i.e. ``expr`` is ``vstack(terms)``. + Row ``j`` of ``expr`` is ``exprs[perm[j]]``. ``None`` if every + expression kept its position, i.e. ``expr`` is ``vstack(exprs)``. Raises ------ ValueError - If ``terms`` is empty or a term does not have shape ``(1, 1)``. + If ``exprs`` is empty or an expression does not have shape ``(1, 1)``. Examples -------- @@ -79,16 +79,17 @@ def compact(terms): >>> perm array([0, 2, 1]) """ - if not terms: - raise ValueError("compact: need at least one term") - values, groups = {}, {} # values: id -> Variable/Parameter the terms use - for i, t in enumerate(terms): + if not exprs: + raise ValueError("compact: need at least one expression") + values, groups = {}, {} # values: id -> Variable/Parameter the exprs use + for i, t in enumerate(exprs): t = cast_to_const(t) if t.shape != (1, 1): - raise ValueError(f"compact: term {i} has shape {t.shape}, not (1, 1)") + raise ValueError(f"compact: expression {i} has shape {t.shape}, " + "not (1, 1)") try: instructions, args = _lower(t, values) - except (UnsupportedTermError, RecursionError): + except (UnsupportedExprError, RecursionError): instructions, args = ("single", i), None groups.setdefault(instructions, []).append((i, t, args)) @@ -103,23 +104,23 @@ def compact(terms): return expr, perm -def _reduce(term): +def _reduce(expr): """Rewrite a sum of many operands as ``sum(compact(operands))``. - The chain of ``add`` nodes in ``term`` is flattened into its operands, + The chain of ``add`` nodes in ``expr`` is flattened into its operands, which are compacted and summed. This reassociates the sum. Parameters ---------- - term : Expression - A scalar term that merged with no other term. + expr : Expression + A scalar expression that merged with no other. Returns ------- Expression - The rewritten sum if it has fewer nodes than ``term``, else ``term``. + The rewritten sum if it has fewer nodes than ``expr``, else ``expr``. """ - operands, stack = [], [term] + operands, stack = [], [expr] while stack: e = stack.pop() if isinstance(e, Node) and e.opcode == "add": @@ -127,9 +128,9 @@ def _reduce(term): else: operands.append(e) if len(operands) < 2: - return term + return expr reduced = atoms.sum(compact(operands)[0]) - return reduced if _count_nodes(reduced) < _count_nodes(term) else term + return reduced if _count_nodes(reduced) < _count_nodes(expr) else expr def _count_nodes(e): @@ -154,34 +155,34 @@ def _count_nodes(e): return len(seen) -def _lower(term, values): - """Lower a scalar term to a list of instructions and its per-term args. +def _lower(expr, values): + """Lower a scalar expression to a list of instructions and its args. - Two terms with equal ``instructions`` apply the same operations and differ + Two expressions with equal ``instructions`` apply the same operations and differ only in their ``args``, so they can be merged by :func:`_vectorize`. Parameters ---------- - term : Expression - A scalar term of shape ``(1, 1)``. + expr : Expression + A scalar expression of shape ``(1, 1)``. values : dict - Maps ``id`` to each Variable or Parameter the terms use. Updated in - place with the ones ``term`` uses. + Maps ``id`` to each Variable or Parameter the exprs use. Updated in + place with the ones ``expr`` uses. Returns ------- instructions : tuple of tuple - The term's nodes in post-order, each as ``(opcode, *operands)``. + The expression's nodes in post-order, each as ``(opcode, *operands)``. Operands are value numbers (positions in ``instructions``), so a - subexpression shared inside the term is computed once. + subexpression shared inside ``expr`` is computed once. args : list - The per-term data, in instruction order: the index of each ``load`` + The per-expression data, in instruction order: the index of each ``load`` and the value of each ``const``. Raises ------ - UnsupportedTermError - If ``term`` uses an operation that cannot be vectorized. + UnsupportedExprError + If ``expr`` uses an operation that cannot be vectorized. """ instructions, args, number = [], [], {} # number: id(node) -> value number @@ -189,7 +190,7 @@ def visit(e): if id(e) in number: return number[id(e)] if e.shape != (1, 1): - raise UnsupportedTermError + raise UnsupportedExprError if isinstance(e, Constant): inst = ("const",) args.append(float(e.value[0, 0])) @@ -197,7 +198,7 @@ def visit(e): values[id(e)] = e inst = ("global", id(e)) elif not isinstance(e, Node): - raise UnsupportedTermError + raise UnsupportedExprError elif e.opcode == "index" and isinstance(e.args[0], Variable): values[id(e.args[0])] = e.args[0] inst = ("load", id(e.args[0])) @@ -209,38 +210,39 @@ def visit(e): elif e.opcode == "power": inst = ("power", e.attrs["p"], visit(e.args[0])) else: - raise UnsupportedTermError + raise UnsupportedExprError number[id(e)] = len(instructions) instructions.append(inst) return number[id(e)] - visit(term) + visit(expr) return tuple(instructions), args def _vectorize(instructions, args, values): - """Build one vector expression for a group of terms with equal instructions. + """Build one vector expression from expressions with equal instructions. - The instructions are run once on column vectors with one row per term: - each ``load`` becomes a gather of the Variable at every term's index, and - each ``const`` a constant vector of every term's value. + The instructions are run once on column vectors with one row per + expression: each ``load`` becomes a gather of the Variable at every + expression's index, and each ``const`` a constant vector of every + expression's value. Parameters ---------- instructions : tuple of tuple - The instructions shared by the terms, as returned by :func:`_lower`. + The instructions shared by the expressions, as returned by :func:`_lower`. args : list of list - ``args[i]`` is the args of term ``i``, as returned by :func:`_lower`. + ``args[i]`` is the args of expression ``i``, as returned by :func:`_lower`. values : dict Maps ``id`` to each Variable or Parameter the instructions use. Returns ------- Expression - An expression of shape ``(len(args), 1)`` whose row ``i`` is term ``i``. + An expression of shape ``(len(args), 1)``; row ``i`` is expression ``i``. """ n_rows = len(args) - arg_columns = iter(zip(*args)) # one load/const arg across all terms + arg_columns = iter(zip(*args)) # one load/const arg across all expressions vecs = [] # vecs[n]: the vector for value number n for inst in instructions: opcode = inst[0] @@ -263,5 +265,5 @@ def _vectorize(instructions, args, values): vecs.append(vecs[inst[2]] ** inst[1]) else: vecs.append(getattr(atoms, opcode)(vecs[inst[1]])) - # with no load/const (every term is e.g. the scalar y) the result is a scalar + # with no load/const (every expression is e.g. y) the result is a scalar return broadcast(vecs[-1], (n_rows, 1)) From d39e8c84a9ea7fb6ef1a6e582e7725c1614e57bf Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:34:02 -0800 Subject: [PATCH 5/7] Problem: always compact scalar objectives and constraints Like an LLVM pass pipeline, Problem now runs compact on the objective (merging a loop-built sum(terms)) and on each run of consecutive scalar constraints. Merged rows are gathered back into the given order with one index node, so constraint values, Jacobian rows and Lagrange multipliers keep their order. test_vectorize_problems now builds the scalar problems directly with Problem. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/problem.py | 38 +++++++++++++++++++++++- tests/test_vectorize_problems.py | 51 ++++++++++++++++++-------------- 2 files changed, 65 insertions(+), 24 deletions(-) diff --git a/sparsediffpy/problem.py b/sparsediffpy/problem.py index 913e489..f8e5505 100644 --- a/sparsediffpy/problem.py +++ b/sparsediffpy/problem.py @@ -10,7 +10,34 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy.expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const +from sparsediffpy.expression import ( + Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const, + index_flat, +) +from sparsediffpy.vectorize import compact + + +def _compact_in_order(exprs): + """``compact(exprs)`` with its rows gathered back into their given order.""" + expr, perm = compact(exprs) + if perm is None: + return expr + return index_flat(expr, np.argsort(perm), expr.shape) + + +def _compact_constraints(constraints): + """Compact each run of consecutive scalar constraints, keeping row order.""" + out, run = [], [] + for c in constraints + [None]: + if c is not None and c.shape == (1, 1): + run.append(c) + continue + if run: + out.append(_compact_in_order(run)) + run = [] + if c is not None: + out.append(c) + return out def _find_scope(exprs): @@ -99,6 +126,12 @@ class Problem: gradient/jacobian are skipped when the matching forward has not moved. Skipped calls return the cached object, which the problem owns: do not modify it in place. + + Scalar expressions written in a Python loop are merged into vector ones + with :func:`~sparsediffpy.compact`: the objective (e.g. ``sum(terms)``) + and each run of consecutive scalar constraints. Constraint rows keep the + order they were given in. ``objective`` and ``constraints`` hold the + expressions as built in the engine. """ def __init__(self, objective, constraints=(), verbose=False): @@ -106,6 +139,9 @@ def __init__(self, objective, constraints=(), verbose=False): constraints = [cast_to_const(c) for c in constraints] if objective.shape != (1, 1): raise ValueError(f"Objective must be scalar, got {objective.shape}") + objective = compact([objective])[0] + constraints = _compact_constraints(constraints) + self.objective, self.constraints = objective, constraints self._scope = _find_scope([objective, *constraints]) build = _Builder(self._scope) obj_cap = build(objective) diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py index 9760273..962dc8b 100644 --- a/tests/test_vectorize_problems.py +++ b/tests/test_vectorize_problems.py @@ -1,4 +1,4 @@ -"""compact() on benchmark NLPs must match the hand-vectorized formulation.""" +"""Benchmark NLPs written as scalar loops must match their vectorized form.""" import numpy as np import pytest @@ -119,18 +119,17 @@ def scalar(x): def _build(problem): + """The hand-vectorized problem and the one written as scalar loops.""" n, vec, scalar = problem f, g = vec(sp.Scope().Variable(n, 1)) ref = sp.Problem(0.0 if f is None else f, [g]) f_terms, g_terms = scalar(sp.Scope().Variable(n, 1)) - f = None if f_terms is None else sp.compact(f_terms)[0] - g, perm = sp.compact(g_terms) - p = sp.Problem(0.0 if f is None else sp.sum(f), [g]) - return n, ref, p, f, g, perm + p = sp.Problem(0.0 if f_terms is None else sum(f_terms), g_terms) + return n, ref, p -def _assert_same(n, ref, p, perm): - rows = slice(None) if perm is None else perm +def _assert_same(n, ref, p, rows=slice(None)): + """p matches ref, whose constraint rows are ``rows`` of p's.""" for seed in range(3): rng = np.random.default_rng(seed) u = 0.5 + 0.3 * rng.standard_normal(n) @@ -145,34 +144,40 @@ def _assert_same(n, ref, p, perm): np.testing.assert_allclose(a, b, **TOL) +def _graph_sizes(p): + return count_nodes(p.objective), [count_nodes(c) for c in p.constraints] + + @pytest.mark.parametrize("make, size", [ (luksan, 3), (luksan, 12), (luksan, 50), (brusselator, 4), (logreg, 6), (vdp, 1), (vdp, 6), (vdp, 20), ], ids=lambda v: getattr(v, "__name__", str(v))) -def test_compact_matches_vectorized(make, size): - n, ref, p, f, g, perm = _build(make(size)) - assert perm is None - _assert_same(n, ref, p, perm) +def test_scalar_problem_matches_vectorized(make, size): + _assert_same(*_build(make(size))) @pytest.mark.parametrize("make, size", [ (luksan, 10), (brusselator, 3), (logreg, 6), (vdp, 3), ], ids=lambda v: getattr(v, "__name__", str(v))) def test_graph_size_does_not_grow(make, size): - *_, f1, g1, _ = _build(make(size)) - *_, f2, g2, _ = _build(make(2 * size)) - assert count_nodes(g1) == count_nodes(g2) - if f1 is not None and make is not logreg: - assert count_nodes(f1) == count_nodes(f2) + f1, g1 = _graph_sizes(_build(make(size))[2]) + f2, g2 = _graph_sizes(_build(make(2 * size))[2]) + assert g1 == g2 + if make is not logreg: # its objective has a dense row per term + assert f1 == f2 -def test_interleaved_rows_are_grouped(): - n, ref, p, f, g, perm = _build(brusselator(3, interleaved=True)) - assert list(perm) == list(range(0, 18, 2)) + list(range(1, 18, 2)) - _assert_same(n, ref, p, None) +def test_interleaved_rows_keep_their_order(): + n, ref, p = _build(brusselator(3, interleaved=True)) + assert len(p.constraints) == 1 + M = 9 # fu_k, fv_k alternate; ref stacks all fu, then all fv + _assert_same(n, ref, p, np.arange(2 * M).reshape(2, M).T.ravel()) -def test_compact_shrinks_the_graph(): +def test_problem_shrinks_the_graph(): n, _, scalar = luksan(200) - terms = scalar(sp.Scope().Variable(n, 1))[1] - assert 50 * count_nodes(sp.compact(terms)[0]) < count_nodes(sp.vstack(terms)) + f_terms, g_terms = scalar(sp.Scope().Variable(n, 1)) + p = sp.Problem(sum(f_terms), g_terms) + f, g = _graph_sizes(p) + assert 50 * f < count_nodes(sum(f_terms)) + assert 50 * sum(g) < count_nodes(sp.vstack(g_terms)) From 5ebf6b9d028e0b41563e798450f06b6c145a54d7 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:45:16 -0800 Subject: [PATCH 6/7] Tests: call compact's inputs exprs, matching the docstrings Co-Authored-By: Claude Opus 5.5 --- tests/test_vectorize.py | 36 ++++++++++++++++---------------- tests/test_vectorize_problems.py | 14 ++++++------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/test_vectorize.py b/tests/test_vectorize.py index 22cb72d..23f414b 100644 --- a/tests/test_vectorize.py +++ b/tests/test_vectorize.py @@ -1,4 +1,4 @@ -"""compact(terms) must merge what it can and match vstack(terms) exactly.""" +"""compact(exprs) must merge what it can and match vstack(exprs) exactly.""" import numpy as np import pytest @@ -17,20 +17,20 @@ def _evaluate(g, u, lam): def assert_compact(build, n, rng): - """build(scope, n) -> scalar terms over n + extra variables. + """build(scope, n) -> scalar expressions over n + extra variables. - The compacted graph must not grow with n (the terms were merged), and - value, gradient, Jacobian and Hessian must match vstack(terms), with rows + The compacted graph must not grow with n (the expressions were merged), and + value, gradient, Jacobian and Hessian must match vstack(exprs), with rows mapped by perm. """ assert count_nodes(sp.compact(build(sp.Scope(), n))[0]) == \ count_nodes(sp.compact(build(sp.Scope(), 2 * n))[0]) scope = sp.Scope() - terms = build(scope, n) + exprs = build(scope, n) u = rng.uniform(0.5, 1.5, scope.total_var_size) - lam = rng.standard_normal(len(terms)) - f, grad, c, J, H = _evaluate(sp.vstack(terms), u, lam) + lam = rng.standard_normal(len(exprs)) + f, grad, c, J, H = _evaluate(sp.vstack(exprs), u, lam) g, perm = sp.compact(build(sp.Scope(), n)) rows = slice(None) if perm is None else perm @@ -57,17 +57,17 @@ def build(scope, n): assert_compact(build, 5, rng) -def test_interleaved_groups_and_unsupported_terms(rng): +def test_interleaved_groups_and_unsupported_exprs(rng): A = rng.standard_normal((2, 6)) def build(scope, n): x, z = scope.Variable(n + 1, 1), scope.Variable(6, 1) - terms = [] + exprs = [] for k in range(n): - terms += [sp.sin(x[k]) * x[k + 1], x[k] ** 2 - 2.0 * k] - terms += [sp.exp(Constant(A[:1]) @ z), + exprs += [sp.sin(x[k]) * x[k + 1], x[k] ** 2 - 2.0 * k] + exprs += [sp.exp(Constant(A[:1]) @ z), sp.exp(Constant(sps.csr_matrix(A[1:])) @ z)] - return terms + return exprs perm = assert_compact(build, 5, rng) assert list(perm) == [0, 2, 4, 6, 8, 1, 3, 5, 7, 9, 10, 11] @@ -76,15 +76,15 @@ def test_shared_subexpression_and_parameter(rng): def build(scope, n): x = scope.Variable(n + 1, 1) p = scope.Parameter(1, 1, value=2.5) - terms = [] + exprs = [] for k in range(n): d = x[k] - x[k + 1] - terms.append(p * d * d + sp.tanh(d)) - return terms + exprs.append(p * d * d + sp.tanh(d)) + return exprs assert_compact(build, 4, rng) -def test_terms_without_args(rng): +def test_exprs_without_args(rng): def build(scope, n): y = scope.Variable(1, 1) return [y] * n + [sp.exp(y) for _ in range(n)] @@ -94,8 +94,8 @@ def build(scope, n): def test_different_structure_does_not_merge(rng): scope = sp.Scope() x, z = scope.Variable(3, 1), scope.Variable(3, 1) - terms = [x[0] ** 2, x[1] ** 2, z[0] ** 2, x[2] ** 3, x[0] * x[1], x[1] * x[1]] - g, perm = sp.compact(terms) + exprs = [x[0] ** 2, x[1] ** 2, z[0] ** 2, x[2] ** 3, x[0] * x[1], x[1] * x[1]] + g, perm = sp.compact(exprs) assert perm is None a, b, c, d = rng.uniform(0.5, 1.5, 4) u = np.array([a, b, c, d, 0.0, 0.0]) diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py index 962dc8b..af630ad 100644 --- a/tests/test_vectorize_problems.py +++ b/tests/test_vectorize_problems.py @@ -123,8 +123,8 @@ def _build(problem): n, vec, scalar = problem f, g = vec(sp.Scope().Variable(n, 1)) ref = sp.Problem(0.0 if f is None else f, [g]) - f_terms, g_terms = scalar(sp.Scope().Variable(n, 1)) - p = sp.Problem(0.0 if f_terms is None else sum(f_terms), g_terms) + f_exprs, g_exprs = scalar(sp.Scope().Variable(n, 1)) + p = sp.Problem(0.0 if f_exprs is None else sum(f_exprs), g_exprs) return n, ref, p @@ -163,7 +163,7 @@ def test_graph_size_does_not_grow(make, size): f1, g1 = _graph_sizes(_build(make(size))[2]) f2, g2 = _graph_sizes(_build(make(2 * size))[2]) assert g1 == g2 - if make is not logreg: # its objective has a dense row per term + if make is not logreg: # its objective uses a dense row of A per expression assert f1 == f2 @@ -176,8 +176,8 @@ def test_interleaved_rows_keep_their_order(): def test_problem_shrinks_the_graph(): n, _, scalar = luksan(200) - f_terms, g_terms = scalar(sp.Scope().Variable(n, 1)) - p = sp.Problem(sum(f_terms), g_terms) + f_exprs, g_exprs = scalar(sp.Scope().Variable(n, 1)) + p = sp.Problem(sum(f_exprs), g_exprs) f, g = _graph_sizes(p) - assert 50 * f < count_nodes(sum(f_terms)) - assert 50 * sum(g) < count_nodes(sp.vstack(g_terms)) + assert 50 * f < count_nodes(sum(f_exprs)) + assert 50 * sum(g) < count_nodes(sp.vstack(g_exprs)) From 5c03238a429c88d5e7b892d0e7ee2397da214989 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:46:42 -0800 Subject: [PATCH 7/7] Tests: keep count_calls docstring within 85 columns Co-Authored-By: Claude Opus 5.5 --- tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index 00e879a..7355665 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -278,7 +278,7 @@ def test_unset_parameter_raises(scope): # --- Caching ------------------------------------------------------------------ def count_calls(monkeypatch, name): - """Count calls to the engine function `name` as seen from sparsediffpy.problem.""" + """Count calls to engine function `name` as seen from sparsediffpy.problem.""" calls, real = [], getattr(problem._C, name) def counted(*args):