diff --git a/doc/advanced/register-vm.md b/doc/advanced/register-vm.md index a9a50a1..ba71236 100644 --- a/doc/advanced/register-vm.md +++ b/doc/advanced/register-vm.md @@ -16,28 +16,38 @@ compiler, or `loadstring` to execute the original source. - `vm/optimize.lua`: recursive, conservative basic-block optimizations and cleanup batching before emission. - `vm/opcodes.lua`: semantic instruction IDs. -- `vm/layout.lua`: a per-emission opcode and operand layout. +- `vm/layout.lua`: per-emission opcode, operand, and numeric metadata keys. - `vm/runtime.lua` and `vm/runtime_dispatch.lua`: runtime templates and the generated dispatcher, configured for the selected opcode IDs and operand positions. - `vm/runtime_calls.lua`: fixed-arity and packed call, return, and register cleanup handlers used by the generated runtime. -- `vm/emit.lua`: instruction encoding, native prototype table literals, and the - runtime function IIFE. +- `vm/data.lua`: scattered table graph encoding and startup reconstruction. +- `vm/emit.lua`: shard literals, the reconstruction helper, and the runtime + function IIFE. - `vm/strings.lua`: optional hoisting of generated interpreter string literals into an outer initializer. - `vm/flatten.lua`: the Cff integration for generated VM interpreters. A prototype contains instructions, constants, child prototypes, parameter -metadata, and capture descriptors. The emitter passes the native prototype -tables as an argument to a runtime function IIFE. The generated interpreter -then uses that table data; the prototypes are not encoded as strings. +metadata, and capture descriptors. Prototype fields use unique randomized +numeric keys instead of fixed string keys. The emitter scatters assignments +for the complete table graph across three to five native shard tables, +including individual instruction operands and constant entries. Table IDs and +record order are shuffled. A startup helper allocates all destination tables, +resolves references, and passes the reconstructed root to the runtime function +IIFE. It releases the consumed shards. Reconstruction happens once per chunk +execution; the interpreter uses ordinary direct table lookups afterward. +Scattering increases output size and startup work without adding shard lookups +to the VM loop. The prototypes are not encoded as strings. Each invocation owns its register table; vararg functions also carry a counted vararg tuple. Closures retain only their referenced upvalue cells; leaving a scope detaches its cells without -destroying values shared by live closures. Value lists carry an explicit `n` -so trailing nils survive calls, varargs, table constructors, and returns. +destroying values shared by live closures. Value lists carry an explicit count +under a per-emission negative numeric key, keeping it separate from positive +array indices so trailing nils survive calls, varargs, table constructors, and +returns. The compiler keeps ordinary scalar values in individual registers. Small calls and returns pass those registers directly to native Lua instead of first making a value-list table. Temporary registers are cleared after consumption, and @@ -66,22 +76,24 @@ initialization path, which also builds the vararg tuple when needed. `Cff` normally flattens control flow throughout its input AST. When it receives the AST emitted by `Vmify`, a private `vmRuntime` annotation selects a special path: by default, `vm/flatten.lua` flattens the interpreter function only. The -large native prototype table initialization remains outside the flattened -function, keeping it out of the Cff control structure and jump limits used by -Lua and LuaJIT. +large native shard initialization and startup reconstruction remain outside +the flattened function, keeping them out of the Cff control structure and jump +limits used by Lua and LuaJIT. Set `Cff`'s `ExcludeVm` setting to `true` to leave both the generated -interpreter and the native prototype table literal out of flattening. Cff then -flattens the surrounding code, including helpers introduced by intervening -steps such as string decoders or tamper checks. The interpreter remains native -Lua, so its lexical captures continue to share variables with surrounding +interpreter and the native shard literal out of flattening. Cff then flattens +the startup reconstruction helper and its call along with surrounding code, +including helpers introduced by intervening steps such as string decoders or +tamper checks. The interpreter remains native Lua, so its lexical captures +continue to share variables with surrounding closures according to ordinary Lua upvalue semantics. `cff/native.lua` emits the excluded expressions directly and bridges their external locals through Cff's shared upvalue cells. Garbage-collected capture proxies keep those cells alive for native closures; captures are not value -snapshots. The `vmData` annotation identifies the entire prototype initializer, -including its child prototypes. +snapshots. The `vmData` annotation identifies only the shard literal containing +the complete prototype graph. It does not exclude the reconstruction helper +or its invocation from Cff. With the default `ExcludeVm = false` behavior, the special path passes read-only external lexical helpers through function arguments. This supports @@ -94,8 +106,9 @@ arity and cleanup instructions use the same ID scheme. At emission, `layout.lua` chooses a permutation of the opcode IDs and a permutation of operand slots 2 through 5; slot 1 always contains the opcode. The emitter encodes each instruction with that layout without changing the compiler -prototypes. One layout is shared by the root prototype and all child -prototypes in the emitted chunk. +prototypes. The layout also selects numeric prototype field keys and the +packed-value count key. One layout is shared by the root prototype and all +child prototypes in the emitted chunk. Before emission, `optimize.lua` visits the root prototype and its children and applies conservative basic-block-local rewrites. At emission, @@ -111,15 +124,17 @@ the generated interpreter's literal strings into an outer initializer, where normal string encryption and later `ConstantArray` processing apply. The initializer decodes them once at startup and binds them to native lexical locals used by the interpreter, avoiding decoder or wrapper calls on hot -paths. Strings in `vmData` prototype tables remain encrypted by the normal +paths. Strings in the `vmData` shards remain encrypted by the normal `EncryptStrings` behavior. `vm/strings.lua` implements this transformation. Medium then applies `AntiTamper`, `ConstantArray`, and `WrapInFunction`, -followed by `Cff` with `ExcludeVm = true`. The interpreter and prototype table -initializers stay native while Cff flattens the surrounding code and generated -helpers. `NumbersToExpressions` runs after Cff: it adds no helper functions, -and this ordering prevents Cff from virtualizing arithmetic constant -expressions. +followed by `Cff` with `ExcludeVm = true`. The interpreter and shard literal +stay native while Cff protects startup reconstruction, surrounding code, and +generated helpers. After generating its helpers, Cff puts repeated string +literals in a shared outer table, including any new diagnostics and counted-value +keys. Each value is embedded only once in Medium output. `NumbersToExpressions` +runs after Cff: it adds no helper functions, and this ordering prevents Cff +from virtualizing arithmetic constant expressions. `Strong` applies one `Vmify` pass followed by `EncryptStrings`, `AntiTamper`, and one default `Cff` pass over the generated VM interpreter. It retains the diff --git a/doc/reference/steps/cff.md b/doc/reference/steps/cff.md index 5ce7632..a93f9ff 100644 --- a/doc/reference/steps/cff.md +++ b/doc/reference/steps/cff.md @@ -8,6 +8,15 @@ When `Cff` receives `Vmify` output, it recognizes the private VM runtime annotat Set `ExcludeVm` to `true` to leave the generated interpreter and native prototype table literal out of Cff flattening. Cff then flattens the surrounding code, including helpers such as decoders and tamper checks, and the outer string initializer emitted by `EncryptStrings` when `HoistVmStrings` is enabled. The interpreter keeps its native Lua lexical captures, preserving shared variable behavior between the interpreter and surrounding closures. +After generating its runtime and helpers, Cff shares repeated string literals +through one outer table with shuffled slots. Each repeated value is embedded +once and reused through indexed references, including diagnostics and counted +argument/result keys introduced by Cff itself. Unique literals stay in place. +This also applies to literals inside preserved native VM expressions; their +control flow stays native. Sharing does not encrypt the remaining literal. +Later steps may introduce new strings. Medium only runs `NumbersToExpressions` +after Cff, so its final output retains one literal per distinct string value. + ## Settings | Setting | Type | Default | Description | diff --git a/doc/reference/steps/vmify.md b/doc/reference/steps/vmify.md index 6dcc98a..9fb22d3 100644 --- a/doc/reference/steps/vmify.md +++ b/doc/reference/steps/vmify.md @@ -10,6 +10,16 @@ slot 1 always stores the opcode. The same layout applies to the root function and its child prototypes. For the same input, configuration, and Lua runtime, a fixed pipeline seed reproduces the layout. +Prototype fields and packed-value counts use per-emission numeric keys; +Vmify introduces no fixed string keys for these internal tables. The complete +prototype graph, including instructions and constants, is scattered across +three to five tables with shuffled records and table IDs. A generated helper +reconstructs ordinary tables once at chunk startup and releases the shards, +leaving direct lookups during VM execution. This adds output size and startup +work. In Medium, Cff protects the reconstruction helper and its invocation; +only the interpreter and large shard literal remain excluded from flattening. +Source-program strings and keys retain their normal semantics. + ## Settings None. diff --git a/src/prometheus/cff/strings.lua b/src/prometheus/cff/strings.lua new file mode 100644 index 0000000..ebdda11 --- /dev/null +++ b/src/prometheus/cff/strings.lua @@ -0,0 +1,46 @@ +-- Share repeated literals after Cff has introduced its own helper strings. +local Ast = require('prometheus.ast') +local visitast = require('prometheus.visitast') +local shuffle = require('prometheus.util').shuffle +local Kind = Ast.AstKind + +return function(ast) + local counts, strings = {}, {} + visitast(ast, function(node) + if node.kind == Kind.StringExpression then + local value = node.value + counts[value] = (counts[value] or 0) + 1 + if counts[value] == 2 then strings[#strings + 1] = value end + end + end) + if #strings == 0 then return ast end + + shuffle(strings) + local scope = ast.body.scope + local pool = scope:addVariable() + local indices, entries = {}, {} + for i, value in ipairs(strings) do + indices[value] = i + entries[i] = Ast.TableEntry(Ast.StringExpression(value)) + end + -- Track blocks explicitly: strings after a nested function literal still + -- belong to the enclosing block, regardless of the visitor's last scope. + local scopes = {} + visitast(ast, function(node) + if node.kind == Kind.Block then scopes[#scopes + 1] = node.scope end + end, function(node) + if node.kind == Kind.Block then + scopes[#scopes] = nil + elseif node.kind == Kind.StringExpression then + local index = indices[node.value] + if index then + scopes[#scopes]:addReferenceToHigherScope(scope, pool) + return Ast.IndexExpression(Ast.VariableExpression(scope, pool), Ast.NumberExpression(index)) + end + end + end) + -- Insert only after replacement so each pooled value remains embedded once. + table.insert(ast.body.statements, 1, + Ast.LocalVariableDeclaration(scope, {pool}, {Ast.TableConstructorExpression(entries)})) + return ast +end diff --git a/src/prometheus/steps/Cff.lua b/src/prometheus/steps/Cff.lua index bc19701..862c383 100644 --- a/src/prometheus/steps/Cff.lua +++ b/src/prometheus/steps/Cff.lua @@ -7,6 +7,7 @@ local Step = require("prometheus.step"); local Compiler = require("prometheus.cff.compiler"); local flattenVm = require("prometheus.vm.flatten"); +local shareStrings = require("prometheus.cff.strings"); local Cff = Step:extend(); Cff.Description = "Flattens control flow into Lua statements executed by a randomized block dispatcher."; @@ -26,15 +27,15 @@ function Cff:apply(ast) local flattened = compiler:compile(ast); flattened.vmRuntime = compiler.nativeResults[ast.vmRuntime]; flattened.vmData = compiler.nativeResults[ast.vmData]; - return flattened; + return shareStrings(flattened); end if ast.vmRuntime then local flattened = flattenVm(ast); - if flattened then return flattened; end + if flattened then return shareStrings(flattened); end end local compiler = Compiler:new(); - return compiler:compile(ast); + return shareStrings(compiler:compile(ast)); end return Cff; diff --git a/src/prometheus/vm/data.lua b/src/prometheus/vm/data.lua new file mode 100644 index 0000000..1e058f1 --- /dev/null +++ b/src/prometheus/vm/data.lua @@ -0,0 +1,82 @@ +-- Scatter the emitted table graph; rebuild ordinary tables once before VM startup. +local Ast = require('prometheus.ast') +local Unparser = require('prometheus.unparser') +local Enums = require('prometheus.enums') +local shuffle = require('prometheus.util').shuffle +local Data = {} + +Data.rebuild = [=[ +local function rebuild(shards, count, root) + local tables = {} + for i = 1, count do tables[i] = {} end + for i = 1, #shards do + local shard = shards[i] + for j = 1, #shard do + local record = shard[j] + local value = record[3] + if record[4] then value = tables[value] end + tables[record[1]][record[2]] = value + end + shards[i] = nil + end + return tables[root] +end +]=] + +function Data.encode(proto, layout) + local function prototype(value) + local result, code, children = {}, {}, {} + for i, instruction in ipairs(value.code) do + code[i] = layout:encode(instruction) + end + for i, child in ipairs(value.children) do children[i] = prototype(child) end + result[layout.fields.code] = code + result[layout.fields.children] = children + for _, name in ipairs({'constants', 'captures', 'params', 'vararg', 'maxRegister'}) do + result[layout.fields[name]] = value[name] + end + return result + end + local root = {prototype(proto)} + local tables, indices, keys = {}, {}, {} + local function collect(value) + if indices[value] then return end + tables[#tables + 1] = value + indices[value] = #tables + local sorted = {} + for key in pairs(value) do sorted[#sorted + 1] = key end + table.sort(sorted) + keys[value] = sorted + for _, key in ipairs(sorted) do + if type(value[key]) == 'table' then collect(value[key]) end + end + end + collect(root) + local ids = {} + for i = 1, #tables do ids[i] = i end + shuffle(ids) + local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local records = {} + for i, value in ipairs(tables) do + for _, key in ipairs(keys[value]) do + local entry, reference = value[key], '' + if type(entry) == 'table' then + entry, reference = ids[indices[entry]], ',1' + end + local literal = unparser:unparseExpression(Ast.ConstantNode(entry), 0) + records[#records + 1] = '{' .. ids[i] .. ',' .. key .. ',' .. literal .. reference .. '}' + end + end + shuffle(records) + local shards = {} + local count = math.random(3, 5) + for i = 1, count do shards[i] = {} end + for i, record in ipairs(records) do + local shard = shards[(i - 1) % count + 1] + shard[#shard + 1] = record + end + for i, shard in ipairs(shards) do shards[i] = '{' .. table.concat(shard, ',') .. '}' end + return '{' .. table.concat(shards, ',') .. '}', #tables, ids[indices[root]] +end + +return Data diff --git a/src/prometheus/vm/emit.lua b/src/prometheus/vm/emit.lua index 619fcc4..19c11ca 100644 --- a/src/prometheus/vm/emit.lua +++ b/src/prometheus/vm/emit.lua @@ -1,50 +1,26 @@ --- Emit native table data and a generated interpreter, then parse the complete AST. -local Ast = require('prometheus.ast') +-- Emit scattered data, startup reconstruction, and a generated interpreter. local Parser = require('prometheus.parser') -local Unparser = require('prometheus.unparser') local Enums = require('prometheus.enums') local Runtime = require('prometheus.vm.runtime') local Layout = require('prometheus.vm.layout') +local Data = require('prometheus.vm.data') local Emit = {} function Emit.source(proto, layout) layout = layout or Layout.new() - local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) - local function literal(value) - if type(value) ~= 'table' then - return unparser:unparseExpression(Ast.ConstantNode(value), 0) - end - local parts = {} - for _, entry in ipairs(value) do parts[#parts + 1] = literal(entry) end - return '{' .. table.concat(parts, ',') .. '}' - end - local function prototype(value) - local instructions = {} - for _, instruction in ipairs(value.code) do - local encoded, length = layout:encode(instruction) - local operands = {} - for slot = 1, length do operands[slot] = literal(encoded[slot]) end - instructions[#instructions + 1] = '{' .. table.concat(operands, ',') .. '}' - end - local parts = {'code={' .. table.concat(instructions, ',') .. '}'} - for _, key in ipairs({'constants', 'captures', 'params', 'vararg', 'maxRegister'}) do - parts[#parts + 1] = key .. '=' .. literal(value[key]) - end - local children = {} - for _, child in ipairs(value.children) do children[#children + 1] = prototype(child) end - parts[#parts + 1] = 'children={' .. table.concat(children, ',') .. '}' - return '{' .. table.concat(parts, ',') .. '}' - end - local data = '{' .. prototype(proto) .. '}' - local source, count = Runtime.build(layout):gsub('__PROTOTYPES__', function() return data end) + local data, tables, root = Data.encode(proto, layout) + local initializer = 'rebuild(' .. data .. ',' .. tables .. ',' .. root .. ')' + local source, count = Runtime.build(layout):gsub('__PROTOTYPES__', function() return initializer end) assert(count == 1, 'Runtime template must contain one prototype marker') - return source + return Data.rebuild .. source end function Emit.ast(proto) local ast = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(Emit.source(proto)) - ast.vmRuntime = ast.body.statements[1].args[1].base - ast.vmData = ast.body.statements[1].args[1].args[1] + local invocation = ast.body.statements[#ast.body.statements].args[1] + ast.vmRuntime = invocation.base + -- Only the large shard literal stays native. Cff protects the reconstruction call/helper. + ast.vmData = invocation.args[1].args[1] return ast end diff --git a/src/prometheus/vm/layout.lua b/src/prometheus/vm/layout.lua index c27652a..782b7a3 100644 --- a/src/prometheus/vm/layout.lua +++ b/src/prometheus/vm/layout.lua @@ -1,6 +1,7 @@ --- Choose one instruction encoding for each generated VM. Compiler IR stays semantic. +-- Choose instruction encoding and metadata keys per VM. Compiler IR stays semantic. local opcodes = require('prometheus.vm.opcodes') local shuffle = require('prometheus.util').shuffle +local fields = {'code', 'constants', 'captures', 'params', 'vararg', 'maxRegister', 'children'} local Layout = {} Layout.__index = Layout @@ -18,6 +19,16 @@ function Layout.new() layout.opcodes[entry.name] = ids[i] layout.ids[entry.id] = ids[i] end + layout.fields = {} + local used = {} + for _, name in ipairs(fields) do + local key + repeat key = math.random(1024, 16777215) until not used[key] + used[key] = true + layout.fields[name] = key + end + -- Counted tuples also contain positive array indices, so keep metadata negative. + layout.countKey = -math.random(1024, 16777215) return layout end diff --git a/src/prometheus/vm/runtime.lua b/src/prometheus/vm/runtime.lua index 5b989bc..9b2f4fd 100644 --- a/src/prometheus/vm/runtime.lua +++ b/src/prometheus/vm/runtime.lua @@ -1,5 +1,5 @@ -- Builds the Lua 5.1 runtime used by the bytecode emitter. The emitter replaces --- __PROTOTYPES__ with its native Lua table literal before parsing the source. +-- __PROTOTYPES__ with the startup reconstruction call before parsing the source. local dispatch = require("prometheus.vm.runtime_dispatch") local Layout = require("prometheus.vm.layout") @@ -15,13 +15,13 @@ local nativeTonumber, nativeError = tonumber, error local nativeSelect, nativeIpairs = select, ipairs local function pack(...) - return { n = nativeSelect("#", ...), ... } + return { [__COUNT_KEY__] = nativeSelect("#", ...), ... } end local make local function loop(P, U, Env, R, V) - local code, K = P.code, P.constants + local code, K = P[__FIELD_code__], P[__FIELD_constants__] local pc = 1 local I, opcode, a, b, c, d, x, y, z while true do @@ -36,9 +36,9 @@ end local function start(P, U, Env, ...) local R - local params = P.params + local params = P[__FIELD_params__] local V - if not P.vararg and #params <= 3 then + if not P[__FIELD_vararg__] and #params <= 3 then R = {} local argc = nativeSelect("#", ...) for i = 1, #params do @@ -63,13 +63,13 @@ local function start(P, U, Env, ...) R[param[1]] = value end end - V = { n = 0 } - if P.vararg then + V = { [__COUNT_KEY__] = 0 } + if P[__FIELD_vararg__] then local first = #params + 1 - if incoming.n > #params then - V.n = incoming.n - #params + if incoming[__COUNT_KEY__] > #params then + V[__COUNT_KEY__] = incoming[__COUNT_KEY__] - #params end - for i = 1, V.n do + for i = 1, V[__COUNT_KEY__] do V[i] = incoming[first + i - 1] end end @@ -110,6 +110,9 @@ function Runtime.build(layout) end):gsub("__DISPATCH__", function() return dispatch.build(layout.opcodes) end) + source = source:gsub('__FIELD_(%w+)__', function(name) + return tostring(assert(layout.fields[name], 'Missing VM prototype key')) + end):gsub('__COUNT_KEY__', function() return tostring(layout.countKey) end) return source end diff --git a/src/prometheus/vm/runtime_calls.lua b/src/prometheus/vm/runtime_calls.lua index 563d819..c0d3b8e 100644 --- a/src/prometheus/vm/runtime_calls.lua +++ b/src/prometheus/vm/runtime_calls.lua @@ -5,26 +5,26 @@ local M = {} local bodies = { CALL = [==[ x, y = R[b], R[c] - if nativeGetfenv and x == nativeGetfenv and (y.n == 0 or y[1] == nil or y[1] == 1) then + if nativeGetfenv and x == nativeGetfenv and (y[__COUNT_KEY__] == 0 or y[1] == nil or y[1] == 1) then if d == -1 then - R[a] = { n = 1, Env } + R[a] = { [__COUNT_KEY__] = 1, Env } elseif d == 1 then R[a] = Env end elseif d == -1 then - R[a] = pack(x(nativeUnpack(y, 1, y.n))) + R[a] = pack(x(nativeUnpack(y, 1, y[__COUNT_KEY__]))) elseif d == 1 then - R[a] = x(nativeUnpack(y, 1, y.n)) + R[a] = x(nativeUnpack(y, 1, y[__COUNT_KEY__])) else - x(nativeUnpack(y, 1, y.n)) + x(nativeUnpack(y, 1, y[__COUNT_KEY__])) end ]==], TAILCALL = [==[ x, y = R[a], R[b] - if nativeGetfenv and x == nativeGetfenv and (y.n == 0 or y[1] == nil or y[1] == 1) then + if nativeGetfenv and x == nativeGetfenv and (y[__COUNT_KEY__] == 0 or y[1] == nil or y[1] == 1) then return Env end - return x(nativeUnpack(y, 1, y.n)) + return x(nativeUnpack(y, 1, y[__COUNT_KEY__])) ]==], CLEARS = [==[ for x = 1, #a do @@ -49,7 +49,7 @@ local function callBody(arity) return "x = R[b]\n" .. "if " .. intercept .. " then\n" .. " if d == -1 then\n" - .. " R[a] = { n = 1, Env }\n" + .. " R[a] = { [__COUNT_KEY__] = 1, Env }\n" .. " elseif d == 1 then\n" .. " R[a] = Env\n" .. " end\n" diff --git a/src/prometheus/vm/runtime_dispatch.lua b/src/prometheus/vm/runtime_dispatch.lua index 653a1ca..8ebecbc 100644 --- a/src/prometheus/vm/runtime_dispatch.lua +++ b/src/prometheus/vm/runtime_dispatch.lua @@ -19,21 +19,21 @@ local bodies = { TABLE = [==[R[a] = {}]==], EXPAND = [==[ x = R[c] - for y = 1, x.n do + for y = 1, x[__COUNT_KEY__] do R[a][b + y - 1] = x[y] end ]==], ARGS = [==[ - x = { n = 0 } + x = { [__COUNT_KEY__] = 0 } for _, y in nativeIpairs(b) do - x.n = x.n + 1 - x[x.n] = R[y] + x[__COUNT_KEY__] = x[__COUNT_KEY__] + 1 + x[x[__COUNT_KEY__]] = R[y] end if c ~= 0 then y = R[c] - for d = 1, y.n do - x.n = x.n + 1 - x[x.n] = y[d] + for d = 1, y[__COUNT_KEY__] do + x[__COUNT_KEY__] = x[__COUNT_KEY__] + 1 + x[x[__COUNT_KEY__]] = y[d] end end R[a] = x @@ -42,13 +42,13 @@ local bodies = { VARARG = [==[R[a] = V]==], RETURN = [==[ x = R[a] - return nativeUnpack(x, 1, x.n) + return nativeUnpack(x, 1, x[__COUNT_KEY__]) ]==], CLOSURE = [==[ - x = P.children[b] + x = P[__FIELD_children__][b] y = {} - for d = 1, #x.captures do - local capture = x.captures[d] + for d = 1, #x[__FIELD_captures__] do + local capture = x[__FIELD_captures__][d] if capture[1] == 0 then y[d] = R[capture[2]] else diff --git a/tests/harness/cff-native.lua b/tests/harness/cff-native.lua index 5a9c2eb..535638f 100644 --- a/tests/harness/cff-native.lua +++ b/tests/harness/cff-native.lua @@ -38,6 +38,12 @@ local function applyCff(step, ast, pipeline) local originalNew, compiler = CffCompiler.new CffCompiler.new = function(class, ...) local instance = originalNew(class, ...) + instance.compiledFunctions = {} + local originalCompileFunction = instance.compileFunction + instance.compileFunction = function(self, node, ...) + self.compiledFunctions[node] = true + return originalCompileFunction(self, node, ...) + end local originalCompile = instance.compile instance.compile = function(self, input) local output = originalCompile(self, input) @@ -173,7 +179,7 @@ local structurePipeline = Prometheus.Pipeline:fromConfig({ LuaVersion = 'Lua51', Seed = 4, Steps = mediumSteps, }) local structureAst = structurePipeline.parser:parse(mediumSource) -local cffAstAfterStep, beforeRuntimeCounts, mediumCompiler +local cffAstAfterStep, beforeRuntimeCounts, mediumCompiler, rebuildHelper for _, step in ipairs(structurePipeline.steps) do if step.Name == 'Cff' then beforeRuntimeCounts = controlFlowCounts(structureAst.vmRuntime) @@ -181,8 +187,13 @@ for _, step in ipairs(structurePipeline.steps) do structureAst = cffAstAfterStep else structureAst = step:apply(structureAst, structurePipeline) or structureAst + if step.Name == 'Vmify' then rebuildHelper = structureAst.body.statements[1] end end end +assert(rebuildHelper and mediumCompiler.compiledFunctions[rebuildHelper], + 'Medium did not compile the startup reconstruction helper through Cff') +assert(not mediumCompiler.nativeExpressions[rebuildHelper], + 'Medium incorrectly excluded the startup reconstruction helper') assert(cffAstAfterStep.vmRuntime and cffAstAfterStep.vmData, 'full Medium pipeline lost native VM runtime/data annotations') assert(countKind(cffAstAfterStep, Ast.AstKind.WhileStatement) > 0 diff --git a/tests/harness/cff-strings.lua b/tests/harness/cff-strings.lua new file mode 100644 index 0000000..066c169 --- /dev/null +++ b/tests/harness/cff-strings.lua @@ -0,0 +1,115 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local Parser = require('prometheus.parser') +local Unparser = require('prometheus.unparser') +local visitast = require('prometheus.visitast') +local shareStrings = require('prometheus.cff.strings') +local Steps = require('prometheus.steps') +local Presets = require('src.presets') + +local parser = Parser:new({LuaVersion = 'Lua51'}) +local unparser = Unparser:new({LuaVersion = 'Lua51'}) +local function inventory(ast) + local counts = {} + visitast(ast, function(node) + if node.kind == 'StringExpression' then counts[node.value] = (counts[node.value] or 0) + 1 end + end) + return counts +end +local function unique(ast) + local counts = inventory(ast) + for value, count in pairs(counts) do + assert(count == 1, 'string embedded more than once: ' .. string.format('%q', value)) + end + return counts +end +local function pack(...) return {n = select('#', ...), ...} end + +-- Empty/unique inputs need no pool. Sharing is idempotent. +for _, source in ipairs({'return ...', 'return "single", 7'}) do + local ast = parser:parse(source) + local statements, variables = #ast.body.statements, #ast.body.scope.variables + assert(shareStrings(ast) == ast) + assert(#ast.body.statements == statements and #ast.body.scope.variables == variables) +end +local source = [[ + local object = {message = 'shared', [''] = 'shared'} + return {function() + return function(delta) + object.message = object.message .. delta + return object.message, '', 'shared', 'a\000b', '\"quoted\"' + end + end, 'shared', '', 'a\000b', '\"quoted\"'}, ... +]] +local ast = parser:parse(source) +shareStrings(ast) +local counts = unique(ast) +assert(counts.shared == 1 and counts['a\000b'] == 1 and counts[''] == 1) +local pool = ast.body.statements[1] +assert(pool.kind == 'LocalVariableDeclaration' and #pool.ids == 1) +local statements, variables = #ast.body.statements, #ast.body.scope.variables +shareStrings(ast) +assert(ast.body.statements[1] == pool and #ast.body.statements == statements + and #ast.body.scope.variables == variables, 'sharing must be idempotent') +local pipeline = Prometheus.Pipeline:fromConfig({LuaVersion='Lua51', Seed=17, Steps={}}) +pipeline:renameVariables(ast) +local chunk = assert(loadstring(unparser:unparse(ast))) +local returned = pack(chunk(nil, false, nil)) +assert(returned.n == 4 and returned[2] == nil and returned[3] == false and returned[4] == nil) +local values = returned[1] +assert(values[2] == 'shared' and values[3] == '' and values[4] == 'a\000b' and values[5] == '"quoted"') +local inner = values[1]() +collectgarbage('collect') +local result = pack(inner('!')) +assert(result.n == 5 and result[1] == 'shared!' and result[2] == '' and result[3] == 'shared' + and result[4] == 'a\000b' and result[5] == '"quoted"') +assert(inner('?') == 'shared!?', 'shared pool broke closure state') + +-- One table binding handles more repeated values than Lua's local variable limit. +local entries = {} +for i = 1, 250 do + entries[#entries + 1] = string.format('%q', 'shared_value_' .. i) + entries[#entries + 1] = entries[#entries] +end +local large = parser:parse('return {' .. table.concat(entries, ',') .. '}') +shareStrings(large) +unique(large) +pipeline:renameVariables(large) +local values = assert(loadstring(unparser:unparse(large)))() +assert(#values == 500 and values[1] == 'shared_value_1' and values[500] == 'shared_value_250') + +-- All Cff paths must share newly generated diagnostics and argument-pack keys. +local program = [[ + local function gather(...) return ... end + local total = 0 + for i = 1, 3 do total = total + i end + for i = 1, 2 do total = total + i end + return gather(total, nil, false, ...) +]] +local configurations = { + {{Name='Cff'}}, + {{Name='Vmify'}, {Name='Cff'}}, + {{Name='Vmify'}, {Name='Cff', Settings={ExcludeVm=true}}}, + Presets.Medium.Steps, +} +for _, seed in ipairs({1, 4, 17}) do + for _, steps in ipairs(configurations) do + local output = Prometheus.Pipeline:fromConfig({LuaVersion='Lua51', Seed=seed, Steps=steps}):apply(program) + local counts = unique(parser:parse(output)) + assert(counts["'for' initial value must be a number"] == 1, + 'Cff must retain exactly one copy of its loop diagnostics') + assert(counts.n == 1, 'Cff must share its counted-value key') + local result = pack(assert(loadstring(output))(nil, 'last', nil)) + assert(result.n == 6 and result[1] == 9 and result[2] == nil and result[3] == false + and result[4] == nil and result[5] == 'last' and result[6] == nil) + end +end + +-- Caching error text must preserve failure behavior, not merely success paths. +local badLoop = Steps.Cff:new():apply(parser:parse('for i = ... , 3 do end')) +unique(badLoop) +pipeline:renameVariables(badLoop) +local loop = assert(loadstring(unparser:unparse(badLoop))) +local ok, err = pcall(loop, {}) +assert(not ok and tostring(err):find("'for' initial value must be a number", 1, true)) +print('Cff shared string literals, scope, and Medium output checks passed') diff --git a/tests/harness/vm-data.lua b/tests/harness/vm-data.lua new file mode 100644 index 0000000..334459c --- /dev/null +++ b/tests/harness/vm-data.lua @@ -0,0 +1,142 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local Ast = require('prometheus.ast') +local Parser = require('prometheus.parser') +local Compiler = require('prometheus.vm.compiler') +local Layout = require('prometheus.vm.layout') +local Data = require('prometheus.vm.data') +local Emit = require('prometheus.vm.emit') +local visitast = require('prometheus.visitast') + +local function equal(left, right) + if type(left) ~= type(right) then return false end + if type(left) ~= 'table' then return left == right end + for key, value in pairs(left) do if not equal(value, right[key]) then return false end end + for key in pairs(right) do if left[key] == nil then return false end end + return true +end +local values = {} +for i = 1, 40 do values[i] = tostring(i * 11) end +local source = 'local values={' .. table.concat(values, ',') .. [[} + local flag = false + return function(a, ...) + flag = not flag + return values[1], values[40], flag, a, ... + end +]] +local proto = Compiler:compile(Parser:new({LuaVersion = 'Lua51'}):parse(source)) +local signatures = {} +for _, seed in ipairs({1, 4, 17, 99}) do + math.randomseed(seed) + local layout = Layout.new() + local encoded, count, rootId = Data.encode(proto, layout) + signatures[encoded] = true + math.randomseed(seed) + local repeated = Data.encode(proto, Layout.new()) + assert(encoded == repeated, 'same seed must reproduce the scattered data') + local shards = assert(loadstring('return ' .. encoded))() + assert(#shards >= 3 and #shards <= 5) + local destinations, seen = {}, {} + local minimum, maximum = math.huge, 0 + for shardId, shard in ipairs(shards) do + minimum, maximum = math.min(minimum, #shard), math.max(maximum, #shard) + for _, record in ipairs(shard) do + local destination, key = record[1], record[2] + assert(destination >= 1 and destination <= count and type(key) == 'number') + seen[destination] = seen[destination] or {} + assert(not seen[destination][key], 'scattered assignments must be unique') + seen[destination][key] = true + destinations[destination] = destinations[destination] or {} + destinations[destination][shardId] = true + end + end + assert(maximum - minimum <= 1 and minimum > 0, 'shards must be balanced and populated') + local rebuild = assert(loadstring(Data.rebuild .. 'return rebuild'))() + local root = rebuild(shards, count, rootId) + assert(next(shards) == nil, 'reconstruction must release consumed shards') + local function check(actual, original) + local fieldCount = 0 + for key in pairs(actual) do + assert(type(key) == 'number', 'prototype keys must not contain fixed strings') + fieldCount = fieldCount + 1 + end + assert(fieldCount == 7) + for _, name in ipairs({'constants', 'captures', 'params', 'vararg', 'maxRegister'}) do + assert(equal(actual[layout.fields[name]], original[name]), name .. ' changed during reconstruction') + end + local code = actual[layout.fields.code] + assert(#code == #original.code) + for i, instruction in ipairs(original.code) do + assert(equal(code[i], layout:encode(instruction)), 'instruction changed during reconstruction') + end + local children = actual[layout.fields.children] + assert(#children == #original.children) + for i, child in ipairs(original.children) do check(children[i], child) end + end + check(root[1], proto) + -- Identify the reconstructed root code/constants arrays and check that their + -- entries, rather than only the prototype metadata, came from multiple shards. + local raw = assert(loadstring('return ' .. encoded))() + local byId = {} + for i = 1, count do byId[i] = {} end + for _, shard in ipairs(raw) do + for _, record in ipairs(shard) do + byId[record[1]][record[2]] = record[4] and byId[record[3]] or record[3] + end + end + for _, field in ipairs({'code', 'constants'}) do + local array = byId[rootId][1][layout.fields[field]] + local id + for index, value in ipairs(byId) do if value == array then id = index end end + local spread = 0 + for _ in pairs(destinations[id]) do spread = spread + 1 end + assert(spread > 1, field .. ' entries were not scattered across shards') + end +end +local distinct = 0 +for _ in pairs(signatures) do distinct = distinct + 1 end +assert(distinct > 1, 'different seeds must change scattered data') + +local blankAst = Emit.ast(Compiler:compile(Parser:new({LuaVersion = 'Lua51'}):parse('return 42'))) +local fixed = {code=true, constants=true, captures=true, params=true, vararg=true, + maxRegister=true, children=true, n=true} +visitast(blankAst, function(node) + if node.kind == 'StringExpression' then + assert(not fixed[node.value], 'fixed VM key remains in generated AST: ' .. node.value) + end +end) +assert(blankAst.vmData.kind == 'TableConstructorExpression') +local runtimeWrapper = Ast.TopNode(Ast.Block({Ast.ReturnStatement({blankAst.vmRuntime})}, + blankAst.body.scope), blankAst.globalScope) +visitast(runtimeWrapper, function(node) + if node.kind == 'VariableExpression' and not node.scope.isGlobal then + assert(node.scope:getVariableName(node.id) ~= 'rebuild', 'reconstruction leaked into runtime') + end +end) + +-- Count helper invocations while repeatedly executing a closure from the chunk. +math.randomseed(87123) +local emitted = Emit.source(proto) +math.randomseed(87123) +assert(emitted == Emit.source(proto), 'same seed must reproduce the VM source') +local instrumented, replacements = emitted:gsub('local function rebuild%(shards, count, root%)', + 'local reconstructionCount = 0\nlocal function rebuild(shards, count, root)\nreconstructionCount = reconstructionCount + 1') +assert(replacements == 1) +instrumented = instrumented:gsub('return tables%[root%]', + 'assert(reconstructionCount == 1); return tables[root]') +local closure = assert(loadstring(instrumented))() +local function pack(...) return {n = select('#', ...), ...} end +for i = 1, 5 do + local result = pack(closure(nil, 7, nil)) + assert(result[1] == 11 and result[2] == 440 and result[3] == (i % 2 == 1)) + assert(result.n == 6 and result[4] == nil and result[5] == 7 and result[6] == nil) +end + +for _, preset in ipairs({'Vmify', 'Medium', 'Strong'}) do + local config = require('src.presets')[preset] + local pipelineConfig = {LuaVersion='Lua51', Seed=17, Steps=config.Steps} + local first = Prometheus.Pipeline:fromConfig(pipelineConfig):apply(source) + local fn = assert(loadstring(first))() + assert(select('#', fn(nil, 7, nil)) == 6, preset .. ' changed trailing nil results') +end +print('VM scattered data, randomized keys, and reconstruction checks passed') diff --git a/tests/harness/vm-layout.lua b/tests/harness/vm-layout.lua index 4423b1d..a5fe1af 100644 --- a/tests/harness/vm-layout.lua +++ b/tests/harness/vm-layout.lua @@ -21,6 +21,10 @@ local function signature(layout) end for id = 1, #names do parts[#parts + 1] = tostring(layout.ids[id]) end for i = 1, 4 do parts[#parts + 1] = tostring(layout.operands[i]) end + for _, name in ipairs({'code', 'constants', 'captures', 'params', 'vararg', 'maxRegister', 'children'}) do + parts[#parts + 1] = tostring(layout.fields[name]) + end + parts[#parts + 1] = tostring(layout.countKey) return table.concat(parts, ',') end @@ -39,7 +43,7 @@ end local first = seeded(87123) assert(signature(first) == signature(seeded(87123)), 'layout must repeat for the same seed') -local opcodeLayouts, operandLayouts = {}, {} +local opcodeLayouts, operandLayouts, keyLayouts = {}, {}, {} for _, seed in ipairs({1, 4, 17, 99, 87123, 234567}) do local layout = seeded(seed) local seen = {} @@ -58,12 +62,27 @@ for _, seed in ipairs({1, 4, 17, 99, 87123, 234567}) do for i = 1, 4 do slots[layout.operands[i]] = true end assert(slots[2] and slots[3] and slots[4] and slots[5]) operandLayouts[table.concat(layout.operands, ',')] = true + local fieldKeys, fieldSignature = {}, {} + for _, name in ipairs({'code', 'constants', 'captures', 'params', 'vararg', 'maxRegister', 'children'}) do + local key = layout.fields[name] + assert(type(key) == 'number' and key > 0 and key % 1 == 0) + assert(not fieldKeys[key], 'prototype keys must be unique') + fieldKeys[key] = true + fieldSignature[#fieldSignature + 1] = tostring(key) + end + assert(layout.countKey < 0 and layout.countKey % 1 == 0, + 'packed count key must not collide with positive array indices') + fieldSignature[#fieldSignature + 1] = tostring(layout.countKey) + keyLayouts[table.concat(fieldSignature, ',')] = true end local distinctOpcodes, distinctOperands = 0, 0 for _ in pairs(opcodeLayouts) do distinctOpcodes = distinctOpcodes + 1 end for _ in pairs(operandLayouts) do distinctOperands = distinctOperands + 1 end assert(distinctOpcodes > 1, 'different seeds must change opcode assignments') assert(distinctOperands > 1, 'different seeds must change operand layouts') +local distinctKeys = 0 +for _ in pairs(keyLayouts) do distinctKeys = distinctKeys + 1 end +assert(distinctKeys > 1, 'different seeds must change VM data keys') local source = [[ local function pack(...) return {n = select('#', ...), ...} end