diff --git a/sparsediffpy/__init__.py b/sparsediffpy/__init__.py index 08995f9..0a59ec1 100644 --- a/sparsediffpy/__init__.py +++ b/sparsediffpy/__init__.py @@ -15,12 +15,13 @@ """ from sparsediffpy import _sparsediffengine # noqa: F401 -from sparsediffpy._expression import ( # noqa: F401 +from sparsediffpy.expression import ( # noqa: F401 DimensionError, Expression, Parameter, Scope, Variable, broadcast, index_flat, ) -from sparsediffpy._atoms import ( # noqa: F401 +from sparsediffpy.atoms import ( # noqa: F401 asinh, atan, atan2, atanh, cos, diag_vec, entr, exp, hstack, log, logistic, normal_cdf, power, prod, quad_form, quad_over_lin, rel_entr, reshape, sin, sinh, sum, tan, tanh, trace, vstack, xexp, ) -from sparsediffpy._compile import CompiledExpression, Problem, compile # noqa: F401 +from sparsediffpy.problem import CompiledExpression, Problem, compile # noqa: F401 +from sparsediffpy.vectorize import compact # noqa: F401 diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/atoms.py similarity index 89% rename from sparsediffpy/_atoms.py rename to sparsediffpy/atoms.py index 5bf22a4..b94a187 100644 --- a/sparsediffpy/_atoms.py +++ b/sparsediffpy/atoms.py @@ -4,7 +4,7 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy._expression import ( +from sparsediffpy.expression import ( Constant, Node, Parameter, Variable, _check_shape, _check_symmetric, cast_to_const, ) @@ -24,7 +24,7 @@ def atom(x): x = cast_to_const(x) if restricted: _require_variables(name, x) - return Node(make, [x], x.shape) + return Node(make, [x], x.shape, name) atom.__name__ = atom.__qualname__ = name atom.__doc__ = f"Elementwise {name}." return atom @@ -56,7 +56,7 @@ def atan2(y, x): _require_variables("atan2", y, x) if y.shape != x.shape: raise ValueError(f"atan2: shape mismatch {y.shape} vs {x.shape}") - return Node(_C.make_atan2, [y, x], y.shape) + return Node(_C.make_atan2, [y, x], y.shape, "atan2") def _reduction_shape(x, axis): @@ -73,7 +73,7 @@ def sum(x, axis=None): x = cast_to_const(x) shape = _reduction_shape(x, axis) c_axis = -1 if axis is None else axis - return Node(lambda c: _C.make_sum(c, c_axis), [x], shape) + return Node(lambda c: _C.make_sum(c, c_axis), [x], shape, "sum") def prod(x, axis=None): @@ -84,7 +84,7 @@ def prod(x, axis=None): shape = _reduction_shape(x, axis) make = {None: _C.make_prod, 0: _C.make_prod_axis_zero, 1: _C.make_prod_axis_one}[axis] - return Node(make, [x], shape) + return Node(make, [x], shape, "prod") def reshape(x, d1, d2): @@ -92,14 +92,14 @@ def reshape(x, d1, d2): x, shape = cast_to_const(x), _check_shape(d1, d2) if x.size != shape[0] * shape[1]: raise ValueError(f"Cannot reshape {x.shape} to {shape}") - return Node(lambda c: _C.make_reshape(c, *shape), [x], shape) + return Node(lambda c: _C.make_reshape(c, *shape), [x], shape, "reshape") def trace(x): x = cast_to_const(x) if x.shape[0] != x.shape[1]: raise ValueError(f"trace requires a square matrix, got {x.shape}") - return Node(_C.make_trace, [x], (1, 1)) + return Node(_C.make_trace, [x], (1, 1), "trace") def diag_vec(x): @@ -107,7 +107,7 @@ def diag_vec(x): x = cast_to_const(x) if x.shape[1] != 1: raise ValueError(f"diag_vec requires a column vector, got {x.shape}") - return Node(_C.make_diag_vec, [x], (x.shape[0], x.shape[0])) + return Node(_C.make_diag_vec, [x], (x.shape[0], x.shape[0]), "diag_vec") def hstack(exprs): @@ -117,7 +117,7 @@ def hstack(exprs): raise ValueError(f"hstack: need >=1 args with equal rows, got " f"{[e.shape for e in exprs]}") shape = (exprs[0].shape[0], np.sum([e.shape[1] for e in exprs]).item()) - return Node(lambda *caps: _C.make_hstack(list(caps)), exprs, shape) + return Node(lambda *caps: _C.make_hstack(list(caps)), exprs, shape, "hstack") def vstack(exprs): @@ -127,7 +127,7 @@ def vstack(exprs): raise ValueError(f"vstack: need >=1 args with equal columns, got " f"{[e.shape for e in exprs]}") shape = (np.sum([e.shape[0] for e in exprs]).item(), exprs[0].shape[1]) - return Node(lambda *caps: _C.make_vstack(list(caps)), exprs, shape) + return Node(lambda *caps: _C.make_vstack(list(caps)), exprs, shape, "vstack") def quad_form(x, Q): @@ -147,18 +147,18 @@ def quad_form(x, Q): if Q._flat is not None: _check_symmetric(Q._flat, Q.shape) return Node(lambda p, c: _C.make_quad_form(p, c, "dense", None, n), - [Q, x], (1, 1)) + [Q, x], (1, 1), "quad_form") if not isinstance(Q, Constant): raise TypeError("quad_form: Q must be a constant or a Parameter") V = (Q.value + Q.value.T) / 2 # the engine assumes symmetric Q if not sps.issparse(V): data = np.ascontiguousarray(V).ravel() return Node(lambda c: _C.make_quad_form(None, c, "dense", data, n), - [x], (1, 1)) + [x], (1, 1), "quad_form") V = sps.csr_matrix(V) return Node(lambda c: _C.make_quad_form( None, c, "sparse", V.data, V.indices.astype(np.int32), - V.indptr.astype(np.int32), n, n), [x], (1, 1)) + V.indptr.astype(np.int32), n, n), [x], (1, 1), "quad_form") def quad_over_lin(x, z): @@ -166,7 +166,7 @@ def quad_over_lin(x, z): _require_variables("quad_over_lin", x, z) if z.shape != (1, 1): raise ValueError(f"quad_over_lin: z must be scalar, got {z.shape}") - return Node(_C.make_quad_over_lin, [x, z], (1, 1)) + return Node(_C.make_quad_over_lin, [x, z], (1, 1), "quad_over_lin") def rel_entr(x, y): @@ -181,4 +181,4 @@ def rel_entr(x, y): shape = y.shape else: raise ValueError(f"rel_entr: incompatible shapes {x.shape} and {y.shape}") - return Node(_C.make_rel_entr, [x, y], shape) + return Node(_C.make_rel_entr, [x, y], shape, "rel_entr") diff --git a/sparsediffpy/_expression.py b/sparsediffpy/expression.py similarity index 90% rename from sparsediffpy/_expression.py rename to sparsediffpy/expression.py index 878bfdf..c897e02 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/expression.py @@ -1,7 +1,8 @@ """Expression graph: Scope, leaves (Variable/Parameter/Constant), and operators. -Every non-leaf is a generic ``Node(make, args, shape)``: ``make`` receives the -C capsules of ``args`` and returns the node's own capsule. Shapes are always +Every non-leaf is a generic ``Node(make, args, shape, opcode)``: ``make`` +receives the C capsules of ``args`` and returns the node's own capsule, and +``opcode`` names the operation. Shapes are always 2-D ``(d1, d2)`` and values are flattened column-major, like the C engine. """ @@ -38,7 +39,7 @@ def size(self): @property def T(self): - return Node(_C.make_transpose, [self], self.shape[::-1]) + return Node(_C.make_transpose, [self], self.shape[::-1], "transpose") def __add__(self, other): return _add(self, cast_to_const(other)) @@ -53,7 +54,7 @@ def __rsub__(self, other): return _add(cast_to_const(other), -self) def __neg__(self): - return Node(_C.make_neg, [self], self.shape) + return Node(_C.make_neg, [self], self.shape, "neg") def __mul__(self, other): return _mul(self, cast_to_const(other)) @@ -70,7 +71,8 @@ def __rmatmul__(self, other): def __pow__(self, p): if not isinstance(p, numbers.Real): raise TypeError("Exponent must be a constant number") - return Node(lambda c: _C.make_power(c, float(p)), [self], self.shape) + return Node(lambda c: _C.make_power(c, float(p)), [self], self.shape, + "power", {"p": float(p)}) def __getitem__(self, key): return _index(self, key) @@ -82,12 +84,19 @@ def __iter__(self): class Node(Expression): - """An operation: ``make(*child_capsules)`` builds its C capsule.""" + """An operation: ``make(*child_capsules)`` builds its C capsule. - def __init__(self, make, args, shape): + ``opcode`` names the operation and ``attrs`` holds the parameters that + rewrites such as :func:`compact` read (``idx`` for index, ``p`` for power). + Compilation only uses ``make``. + """ + + def __init__(self, make, args, shape, opcode, attrs=None): self.make = make self.args = tuple(args) self.shape = tuple(shape) + self.opcode = opcode + self.attrs = attrs class Constant(Expression): @@ -254,23 +263,26 @@ def broadcast(x, shape): raise ValueError(f"Cannot broadcast {x.shape} to {shape}") if isinstance(x, Constant): return Constant(np.broadcast_to(x.dense(), shape)) - return Node(lambda c: _C.make_broadcast(c, *shape), [x], shape) + return Node(lambda c: _C.make_broadcast(c, *shape), [x], shape, + "broadcast") def _add(a, b): shape = _broadcast_shape(a.shape, b.shape) - return Node(_C.make_add, [broadcast(a, shape), broadcast(b, shape)], shape) + return Node(_C.make_add, [broadcast(a, shape), broadcast(b, shape)], shape, + "add") def _mul(a, b): if _is_data(b) and not _is_data(a): a, b = b, a # put the constant/parameter operand (if any) first if _is_data(a) and a.shape == (1, 1): - return Node(_C.make_param_scalar_mult, [a, b], b.shape) + return Node(_C.make_param_scalar_mult, [a, b], b.shape, "scalar_mult") shape = _broadcast_shape(a.shape, b.shape) a, b = broadcast(a, shape), broadcast(b, shape) - make = _C.make_param_vector_mult if _is_data(a) else _C.make_multiply - return Node(make, [a, b], shape) + if _is_data(a): + return Node(_C.make_param_vector_mult, [a, b], shape, "vector_mult") + return Node(_C.make_multiply, [a, b], shape, "multiply") def _matmul(a, b): @@ -279,11 +291,11 @@ def _matmul(a, b): shape = (a.shape[0], b.shape[1]) if _is_data(a): return Node(_matmul_with_data(_C.make_left_matmul, a), _data_args(a, b), - shape) + shape, "left_matmul") if _is_data(b): return Node(_matmul_with_data(_C.make_right_matmul, b), _data_args(b, a), - shape) - return Node(_C.make_matmul, [a, b], shape) + shape, "right_matmul") + return Node(_C.make_matmul, [a, b], shape, "matmul") def _data_args(data, x): @@ -312,7 +324,8 @@ def index_flat(x, flat_indices, shape): raise ValueError(f"{idx.size} indices do not fill shape {shape}") if idx.size and (idx.min() < 0 or idx.max() >= x.size): raise IndexError(f"flat index out of range for size {x.size}") - return Node(lambda c: _C.make_index(c, shape[0], shape[1], idx), [x], shape) + return Node(lambda c: _C.make_index(c, shape[0], shape[1], idx), [x], shape, + "index", {"idx": idx}) def _axis_indices(key, n): diff --git a/sparsediffpy/_compile.py b/sparsediffpy/problem.py similarity index 88% rename from sparsediffpy/_compile.py rename to sparsediffpy/problem.py index 8554b10..a5e24e3 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/problem.py @@ -10,7 +10,34 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy._expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const +from sparsediffpy.expression import ( + Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const, + index_flat, +) +from sparsediffpy.vectorize import compact + + +def _compact_in_order(exprs): + """``compact(exprs)`` with its rows gathered back into their given order.""" + expr, perm = compact(exprs) + if perm is None: + return expr + return index_flat(expr, np.argsort(perm), expr.shape) + + +def _compact_constraints(constraints): + """Compact each run of consecutive scalar constraints, keeping row order.""" + out, run = [], [] + for c in constraints + [None]: + if c is not None and c.shape == (1, 1): + run.append(c) + continue + if run: + out.append(_compact_in_order(run)) + run = [] + if c is not None: + out.append(c) + return out def _find_scope(exprs): @@ -99,6 +126,12 @@ class Problem: gradient/jacobian are skipped when the matching forward has not moved. Skipped calls return the cached object, which the problem owns: do not modify it in place. + + Each run of consecutive scalar constraints, such as one per iteration of + a Python loop, is merged into vector expressions with + :func:`~sparsediffpy.compact`. Constraint rows keep the order they were + given in. ``objective`` and ``constraints`` hold the expressions as built + in the engine. """ def __init__(self, objective, constraints=(), verbose=False): @@ -106,6 +139,8 @@ def __init__(self, objective, constraints=(), verbose=False): constraints = [cast_to_const(c) for c in constraints] if objective.shape != (1, 1): raise ValueError(f"Objective must be scalar, got {objective.shape}") + constraints = _compact_constraints(constraints) + self.objective, self.constraints = objective, constraints self._scope = _find_scope([objective, *constraints]) build = _Builder(self._scope) obj_cap = build(objective) diff --git a/sparsediffpy/vectorize.py b/sparsediffpy/vectorize.py new file mode 100644 index 0000000..da17876 --- /dev/null +++ b/sparsediffpy/vectorize.py @@ -0,0 +1,211 @@ +"""Merge scalar expressions written in a Python loop into vector expressions. + +A loop such as ``[x[k] ** 2 + x[k + 1] - 1 for k in range(n - 1)]`` gives the +engine one small expression per term. :func:`compact` lowers each expression +to a short list of instructions, in the style of LLVM IR:: + + %0 = load x ; arg: the index k + %1 = power 2.0 %0 + %2 = load x ; arg: k + 1 + %3 = add %1 %2 + %4 = const ; arg: 1.0 + %5 = neg %4 + %6 = add %3 %5 + +Expressions whose instructions are identical differ only in their args, so each such +group is rebuilt once as a vector expression: a ``load`` becomes a gather of +``x`` at the args, a ``const`` becomes a constant vector. +""" + +import numpy as np + +from sparsediffpy import atoms +from sparsediffpy.expression import ( + Constant, Node, Parameter, Variable, cast_to_const, broadcast, index_flat, +) + +# Unary atoms that accept any argument (restricted ones need plain Variables). +_UNARY = {"sin", "cos", "exp", "sinh", "tanh", "asinh", "atan", "logistic", + "normal_cdf", "xexp"} + + +class UnsupportedExprError(Exception): + """Raised when an expression uses an operation that cannot be vectorized.""" + + +def compact(exprs): + """Stack scalar expressions into a column vector, merging those of the same form. + + ``exprs`` is typically the list of scalar objective or constraint terms + built in a Python loop, such as + ``[x[k] ** 2 + x[k + 1] for k in range(n - 1)]``. Expressions that apply the + same operations, and differ only in which entries of a Variable they index + or which constants they use, are rebuilt as one vector expression. Other + expressions (using matmul, sum, ...) are kept as they are. + + Parameters + ---------- + exprs : list of Expression or float + The scalar expressions to stack; each must have shape ``(1, 1)``. + + Returns + ------- + expr : Expression + An expression of shape ``(len(exprs), 1)`` holding one expression per + row. Merged expressions are grouped together, so rows may be reordered. + perm : numpy.ndarray of int or None + Row ``j`` of ``expr`` is ``exprs[perm[j]]``. ``None`` if every + expression kept its position, i.e. ``expr`` is ``vstack(exprs)``. + + Raises + ------ + ValueError + If ``exprs`` is empty or an expression does not have shape ``(1, 1)``. + + Examples + -------- + >>> x = sp.Scope().Variable(5, 1) + >>> expr, perm = sp.compact([x[k] ** 2 + x[k + 1] for k in range(4)]) + >>> expr.shape, perm + ((4, 1), None) + >>> expr, perm = sp.compact([x[0] ** 2, sp.exp(x[1]), x[2] ** 2]) + >>> perm + array([0, 2, 1]) + """ + if not exprs: + raise ValueError("compact: need at least one expression") + values, groups = {}, {} # values: id -> Variable/Parameter the exprs use + for i, t in enumerate(exprs): + t = cast_to_const(t) + if t.shape != (1, 1): + raise ValueError(f"compact: expression {i} has shape {t.shape}, " + "not (1, 1)") + try: + instructions, args = _lower(t, values) + except (UnsupportedExprError, RecursionError): # deep exprs are kept as is + instructions, args = ("single", i), None + groups.setdefault(instructions, []).append((i, t, args)) + + parts, perm = [], [] + for instructions, members in groups.items(): + perm += [i for i, _, _ in members] + parts.append(members[0][1] if len(members) == 1 else + _vectorize(instructions, + [args for _, _, args in members], values)) + expr = parts[0] if len(parts) == 1 else atoms.vstack(parts) + perm = None if perm == list(range(len(perm))) else np.array(perm) + return expr, perm + + +def _lower(expr, values): + """Lower a scalar expression to a list of instructions and its args. + + Two expressions with equal ``instructions`` apply the same operations and differ + only in their ``args``, so they can be merged by :func:`_vectorize`. + + Parameters + ---------- + expr : Expression + A scalar expression of shape ``(1, 1)``. + values : dict + Maps ``id`` to each Variable or Parameter the exprs use. Updated in + place with the ones ``expr`` uses. + + Returns + ------- + instructions : tuple of tuple + The expression's nodes in post-order, each as ``(opcode, *operands)``. + Operands are value numbers (positions in ``instructions``), so a + subexpression shared inside ``expr`` is computed once. + args : list + The per-expression data, in instruction order: the index of each ``load`` + and the value of each ``const``. + + Raises + ------ + UnsupportedExprError + If ``expr`` uses an operation that cannot be vectorized. + """ + instructions, args, number = [], [], {} # number: id(node) -> value number + + def visit(e): + if id(e) in number: + return number[id(e)] + if e.shape != (1, 1): + raise UnsupportedExprError + if isinstance(e, Constant): + inst = ("const",) + args.append(float(e.value[0, 0])) + elif isinstance(e, (Variable, Parameter)): + values[id(e)] = e + inst = ("global", id(e)) + elif not isinstance(e, Node): + raise UnsupportedExprError + elif e.opcode == "index" and isinstance(e.args[0], Variable): + values[id(e.args[0])] = e.args[0] + inst = ("load", id(e.args[0])) + args.append(int(e.attrs["idx"][0])) + elif e.opcode in ("add", "multiply", "scalar_mult"): + inst = (e.opcode, visit(e.args[0]), visit(e.args[1])) + elif e.opcode == "neg" or e.opcode in _UNARY: + inst = (e.opcode, visit(e.args[0])) + elif e.opcode == "power": + inst = ("power", e.attrs["p"], visit(e.args[0])) + else: + raise UnsupportedExprError + number[id(e)] = len(instructions) + instructions.append(inst) + return number[id(e)] + + visit(expr) + return tuple(instructions), args + + +def _vectorize(instructions, args, values): + """Build one vector expression from expressions with equal instructions. + + The instructions are run once on column vectors with one row per + expression: each ``load`` becomes a gather of the Variable at every + expression's index, and each ``const`` a constant vector of every + expression's value. + + Parameters + ---------- + instructions : tuple of tuple + The instructions shared by the expressions, as returned by :func:`_lower`. + args : list of list + ``args[i]`` is the args of expression ``i``, as returned by :func:`_lower`. + values : dict + Maps ``id`` to each Variable or Parameter the instructions use. + + Returns + ------- + Expression + An expression of shape ``(len(args), 1)``; row ``i`` is expression ``i``. + """ + n_rows = len(args) + arg_columns = iter(zip(*args)) # one load/const arg across all expressions + vecs = [] # vecs[n]: the vector for value number n + for inst in instructions: + opcode = inst[0] + if opcode == "load": + vecs.append(index_flat(values[inst[1]], next(arg_columns), + (n_rows, 1))) + elif opcode == "const": + c = np.array(next(arg_columns)) + vecs.append(Constant(c[0] if np.all(c == c[0]) + else c.reshape(n_rows, 1))) + elif opcode == "global": + vecs.append(values[inst[1]]) + elif opcode == "add": + vecs.append(vecs[inst[1]] + vecs[inst[2]]) + elif opcode in ("multiply", "scalar_mult"): + vecs.append(vecs[inst[1]] * vecs[inst[2]]) + elif opcode == "neg": + vecs.append(-vecs[inst[1]]) + elif opcode == "power": + vecs.append(vecs[inst[2]] ** inst[1]) + else: + vecs.append(getattr(atoms, opcode)(vecs[inst[1]])) + # with no load/const (every expression is e.g. y) the result is a scalar + return broadcast(vecs[-1], (n_rows, 1)) diff --git a/tests/conftest.py b/tests/conftest.py index bd8e38b..1b43a6f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,3 +40,14 @@ def jac(x): scope.set_values(x0) np.testing.assert_allclose(J, J_fd, rtol=rtol, atol=atol, err_msg="Jacobian") np.testing.assert_allclose(H, H_fd, rtol=rtol, atol=atol, err_msg="Hessian") + + +def count_nodes(e): + """Number of distinct nodes in the expression DAG rooted at e.""" + seen, stack = set(), [e] + while stack: + e = stack.pop() + if id(e) not in seen: + seen.add(id(e)) + stack.extend(e.args) + return len(seen) diff --git a/tests/test_api.py b/tests/test_api.py index ec645e6..7355665 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -6,7 +6,7 @@ import pytest import sparsediffpy as sp -from sparsediffpy import _compile +from sparsediffpy import problem # --- scope and values --------------------------------------------------------- @@ -278,13 +278,13 @@ def test_unset_parameter_raises(scope): # --- Caching ------------------------------------------------------------------ def count_calls(monkeypatch, name): - """Count calls to the engine function `name` as seen from _compile.""" - calls, real = [], getattr(_compile._C, name) + """Count calls to engine function `name` as seen from sparsediffpy.problem.""" + calls, real = [], getattr(problem._C, name) def counted(*args): calls.append(1) return real(*args) - monkeypatch.setattr(_compile._C, name, counted) + monkeypatch.setattr(problem._C, name, counted) return calls diff --git a/tests/test_vectorize.py b/tests/test_vectorize.py new file mode 100644 index 0000000..4d9f856 --- /dev/null +++ b/tests/test_vectorize.py @@ -0,0 +1,135 @@ +"""compact(exprs) must merge what it can and match vstack(exprs) exactly.""" + +import numpy as np +import pytest +import scipy.sparse as sps + +import sparsediffpy as sp +from conftest import count_nodes +from sparsediffpy.expression import Constant + + +def _evaluate(g, u, lam): + prob = sp.Problem(sp.sum(g), [g]) + f, c = prob.objective_forward(u), prob.constraint_forward(u) + return (f, prob.gradient().copy(), c.copy(), prob.jacobian().toarray(), + prob.hessian(0.7, lam).toarray()) + + +def assert_compact(build, n, rng): + """build(scope, n) -> scalar expressions over n + extra variables. + + The compacted graph must not grow with n (the expressions were merged), and + value, gradient, Jacobian and Hessian must match vstack(exprs), with rows + mapped by perm. + """ + assert count_nodes(sp.compact(build(sp.Scope(), n))[0]) == \ + count_nodes(sp.compact(build(sp.Scope(), 2 * n))[0]) + + scope = sp.Scope() + exprs = build(scope, n) + u = rng.uniform(0.5, 1.5, scope.total_var_size) + lam = rng.standard_normal(len(exprs)) + f, grad, c, J, H = _evaluate(sp.vstack(exprs), u, lam) + + g, perm = sp.compact(build(sp.Scope(), n)) + rows = slice(None) if perm is None else perm + got = _evaluate(g, u, lam[rows]) + expected = (f, grad, c[rows], J[rows], H) + for name, a, b in zip(["objective", "gradient", "constraints", "jacobian", + "hessian"], expected, got): + np.testing.assert_allclose(b, a, rtol=1e-12, atol=1e-12, err_msg=name) + return perm + + +def test_loop_merges_in_place(rng): + def build(scope, n): + x = scope.Variable(n + 2, 1) + return [3 * x[k + 1] ** 3 + 2 * x[k + 2] - x[k] * sp.exp(x[k] - x[k + 1]) + for k in range(n)] + assert assert_compact(build, 6, rng) is None + + +def test_varying_constants(rng): + def build(scope, n): + x = scope.Variable(n, 1) + return [(k + 1.0) * (x[k] - k) ** 2 for k in range(n)] + assert_compact(build, 5, rng) + + +def test_interleaved_groups_and_unsupported_exprs(rng): + A = rng.standard_normal((2, 6)) + + def build(scope, n): + x, z = scope.Variable(n + 1, 1), scope.Variable(6, 1) + exprs = [] + for k in range(n): + exprs += [sp.sin(x[k]) * x[k + 1], x[k] ** 2 - 2.0 * k] + exprs += [sp.exp(Constant(A[:1]) @ z), + sp.exp(Constant(sps.csr_matrix(A[1:])) @ z)] + return exprs + perm = assert_compact(build, 5, rng) + assert list(perm) == [0, 2, 4, 6, 8, 1, 3, 5, 7, 9, 10, 11] + + +def test_shared_subexpression_and_parameter(rng): + def build(scope, n): + x = scope.Variable(n + 1, 1) + p = scope.Parameter(1, 1, value=2.5) + exprs = [] + for k in range(n): + d = x[k] - x[k + 1] + exprs.append(p * d * d + sp.tanh(d)) + return exprs + assert_compact(build, 4, rng) + + +def test_exprs_without_args(rng): + def build(scope, n): + y = scope.Variable(1, 1) + return [y] * n + [sp.exp(y) for _ in range(n)] + assert_compact(build, 2, rng) + + +def test_different_structure_does_not_merge(rng): + scope = sp.Scope() + x, z = scope.Variable(3, 1), scope.Variable(3, 1) + exprs = [x[0] ** 2, x[1] ** 2, z[0] ** 2, x[2] ** 3, x[0] * x[1], x[1] * x[1]] + g, perm = sp.compact(exprs) + assert perm is None + a, b, c, d = rng.uniform(0.5, 1.5, 4) + u = np.array([a, b, c, d, 0.0, 0.0]) + np.testing.assert_allclose(sp.Problem(sp.sum(g), [g]).constraint_forward(u), + [a * a, b * b, d * d, c ** 3, a * b, b * b]) + + +def test_parameter_update_after_compact(): + scope = sp.Scope() + x = scope.Variable(3, 1) + p = scope.Parameter(1, 1, value=2.0) + g, _ = sp.compact([p * x[k] for k in range(3)]) + prob = sp.Problem(sp.sum(g), [g]) + u = np.array([1.0, 2.0, 3.0]) + np.testing.assert_allclose(prob.constraint_forward(u), 2.0 * u) + p.value = 5.0 + np.testing.assert_allclose(prob.constraint_forward(u), 5.0 * u) + + +def test_deep_expr_is_kept_as_is(): + x = sp.Scope().Variable(3000, 1) + deep = x[0] + for k in range(1, 3000): + deep = deep + x[k] + g, perm = sp.compact([deep, x[0] ** 2, x[1] ** 2]) + assert perm is None and g.args[0] is deep + u = np.ones(3000) + np.testing.assert_allclose(sp.Problem(sp.sum(g), [g]).constraint_forward(u), + [3000.0, 1.0, 1.0]) + + +def test_rejects_non_scalar_and_empty(): + x = sp.Scope().Variable(3, 1) + with pytest.raises(ValueError): + sp.compact([x[0], x]) + with pytest.raises(ValueError): + sp.compact([]) diff --git a/tests/test_vectorize_problems.py b/tests/test_vectorize_problems.py new file mode 100644 index 0000000..e36960a --- /dev/null +++ b/tests/test_vectorize_problems.py @@ -0,0 +1,178 @@ +"""Benchmark NLPs written as scalar loops must match their vectorized form.""" + +import numpy as np +import pytest + +import sparsediffpy as sp +from conftest import count_nodes +from sparsediffpy.expression import Constant + +TOL = dict(rtol=1e-10, atol=1e-10) + + +def luksan(n): + def vec(x): + x0, x1 = x[0:n - 1], x[1:n] + f = sp.sum(100.0 * (x0 ** 2 - x1) ** 2 + (x0 - 1.0) ** 2) + a, b, c = x[0:n - 2], x[1:n - 1], x[2:n] + g = (3.0 * b ** 3 + 2.0 * c - 5.0 + sp.sin(b - c) * sp.sin(b + c) + + 4.0 * b - a * sp.exp(a - b) - 3.0) + return f, g + + def scalar(x): + xs = [x[i] for i in range(n)] + f = [100.0 * (xs[i] ** 2 - xs[i + 1]) ** 2 + (xs[i] - 1.0) ** 2 + for i in range(n - 1)] + g = [3.0 * xs[k + 1] ** 3 + 2.0 * xs[k + 2] - 5.0 + + sp.sin(xs[k + 1] - xs[k + 2]) * sp.sin(xs[k + 1] + xs[k + 2]) + + 4.0 * xs[k + 1] - xs[k] * sp.exp(xs[k] - xs[k + 1]) - 3.0 + for k in range(n - 2)] + return f, g + + return n, vec, scalar + + +def brusselator(N, interleaved=False): + M, alpha = N * N, 10.0 * (N - 1) ** 2 / 1e4 + X, Y = np.meshgrid(np.linspace(0, 1, N), np.linspace(0, 1, N), indexing="ij") + F = 5.0 * (((X - 0.3) ** 2 + (Y - 0.6) ** 2) <= 0.04).ravel() + idx = np.arange(M).reshape(N, N) + nbrs = [np.roll(idx, s, axis=a).ravel() for s in (-1, 1) for a in (0, 1)] + + def vec(x): + u, v = x[0:M], x[M:2 * M] + + def lap(w): + return sum(w[i] for i in nbrs) - 4.0 * w + + u2v = u ** 2 * v + fu = 1.0 + u2v - 4.4 * u + alpha * lap(u) + F.reshape(-1, 1) + fv = 3.4 * u - u2v + alpha * lap(v) + return None, sp.vstack([fu, fv]) + + def scalar(x): + u, v = [x[i] for i in range(M)], [x[M + i] for i in range(M)] + + def lap(w, k): + return sum(w[i[k]] for i in nbrs) - 4.0 * w[k] + + fu = [1.0 + u[k] ** 2 * v[k] - 4.4 * u[k] + alpha * lap(u, k) + F[k] + for k in range(M)] + fv = [3.4 * u[k] - u[k] ** 2 * v[k] + alpha * lap(v, k) + for k in range(M)] + if interleaved: + return None, [t for pair in zip(fu, fv) for t in pair] + return None, fu + fv + + return 2 * M, vec, scalar + + +def logreg(n): + rng = np.random.default_rng(0) + A = rng.standard_normal((2 * n, n)) / np.sqrt(n) + c = A.T @ (rng.random(2 * n) < 0.5) + mu = 1e-2 + + def vec(x): + f = (sp.sum(sp.logistic(Constant(A) @ x)) - Constant(c[None, :]) @ x + + 0.5 * mu * sp.sum(x ** 2)) + return f, x[0:n - 1] ** 2 + x[1:n] ** 2 + + def scalar(x): + xs = [x[i] for i in range(n)] + f = [sp.logistic(Constant(a[None, :]) @ x) for a in A] + f += [-c[j] * xs[j] + 0.5 * mu * xs[j] ** 2 for j in range(n)] + return f, [xs[i] ** 2 + xs[i + 1] ** 2 for i in range(n - 1)] + + return n, vec, scalar + + +def vdp(N): + h = 10.0 / N + + def rk4(a, b, u): + def rhs(a, b): + return (1.0 - b ** 2) * a - b + u, a + + k1 = rhs(a, b) + k2 = rhs(a + 0.5 * h * k1[0], b + 0.5 * h * k1[1]) + k3 = rhs(a + 0.5 * h * k2[0], b + 0.5 * h * k2[1]) + k4 = rhs(a + h * k3[0], b + h * k3[1]) + return [s + (h / 6.0) * (p + 2.0 * q + 2.0 * r + t) + for s, p, q, r, t in zip((a, b), k1, k2, k3, k4)] + + def vec(x): + x1, x2, u = x[0:N + 1], x[N + 1:2 * N + 2], x[2 * N + 2:3 * N + 2] + na, nb = rk4(x1[0:N], x2[0:N], u) + return (sp.sum(x1 ** 2) + sp.sum(x2 ** 2) + sp.sum(u ** 2), + sp.vstack([x1[1:N + 1] - na, x2[1:N + 1] - nb])) + + def scalar(x): + xs = [x[i] for i in range(3 * N + 2)] + x1, x2, u = xs[0:N + 1], xs[N + 1:2 * N + 2], xs[2 * N + 2:] + steps = [rk4(x1[k], x2[k], u[k]) for k in range(N)] + g = ([x1[k + 1] - s[0] for k, s in enumerate(steps)] + + [x2[k + 1] - s[1] for k, s in enumerate(steps)]) + return [t ** 2 for t in xs], g + + return 3 * N + 2, vec, scalar + + +def _build(problem): + """The hand-vectorized problem and the one written as scalar loops.""" + n, vec, scalar = problem + f, g = vec(sp.Scope().Variable(n, 1)) + ref = sp.Problem(0.0 if f is None else f, [g]) + f_exprs, g_exprs = scalar(sp.Scope().Variable(n, 1)) + p = sp.Problem(0.0 if f_exprs is None else sum(f_exprs), g_exprs) + return n, ref, p + + +def _assert_same(n, ref, p, rows=slice(None)): + """p matches ref, whose constraint rows are ``rows`` of p's.""" + for seed in range(3): + rng = np.random.default_rng(seed) + u = 0.5 + 0.3 * rng.standard_normal(n) + lam = rng.standard_normal(ref.total_constraint_size) + for a, b in [ + (p.objective_forward(u), ref.objective_forward(u)), + (p.gradient(), ref.gradient()), + (p.constraint_forward(u), ref.constraint_forward(u)[rows]), + (p.jacobian().toarray(), ref.jacobian().toarray()[rows]), + (p.hessian(1.3, lam[rows]).toarray(), + ref.hessian(1.3, lam).toarray())]: + np.testing.assert_allclose(a, b, **TOL) + + +def _constraint_sizes(p): + return [count_nodes(c) for c in p.constraints] + + +@pytest.mark.parametrize("make, size", [ + (luksan, 3), (luksan, 12), (luksan, 50), (brusselator, 4), (logreg, 6), + (vdp, 1), (vdp, 6), (vdp, 20), +], ids=lambda v: getattr(v, "__name__", str(v))) +def test_scalar_problem_matches_vectorized(make, size): + _assert_same(*_build(make(size))) + + +@pytest.mark.parametrize("make, size", [ + (luksan, 10), (brusselator, 3), (logreg, 6), (vdp, 3), +], ids=lambda v: getattr(v, "__name__", str(v))) +def test_graph_size_does_not_grow(make, size): + p1, p2 = _build(make(size))[2], _build(make(2 * size))[2] + assert _constraint_sizes(p1) == _constraint_sizes(p2) + + +def test_interleaved_rows_keep_their_order(): + n, ref, p = _build(brusselator(3, interleaved=True)) + assert len(p.constraints) == 1 + M = 9 # fu_k, fv_k alternate; ref stacks all fu, then all fv + _assert_same(n, ref, p, np.arange(2 * M).reshape(2, M).T.ravel()) + + +def test_problem_shrinks_the_graph(): + n, _, scalar = luksan(200) + g_exprs = scalar(sp.Scope().Variable(n, 1))[1] + p = sp.Problem(0.0, g_exprs) + assert 50 * sum(_constraint_sizes(p)) < count_nodes(sp.vstack(g_exprs))