#!/usr/bin/env lua package.path = table.concat({ "?.lua", "?/init.lua", "tools/lua/?.lua", "tools/lua/?/init.lua", package.path, }, ";") local json = require("tools.lua.json") local path = require("tools.lua.path") local process = require("tools.lua.process") local M = {} local BACKENDS = { "vulkan", "opengl", "d3d11", "d3d12" } local SCENES = { { id = "stencil_clip", example = "parity_stencil_clip", ledger = "bp-stencil", negative_control = true, required_backends = { "vulkan", "opengl", "d3d11", "d3d12" }, assertions = { { kind = "point", x = 48, y = 48, expected = "background" }, { kind = "point", x = 128, y = 128, expected = "foreground" }, { kind = "rect", x = 0, y = 0, w = 32, h = 256, expected = "background" }, }, }, { id = "resource_sync", example = "parity_resource_sync", ledger = "bp-resource-sync", negative_control = false, required_backends = { "vulkan", "opengl", "d3d11", "d3d12" }, assertions = { { kind = "point", x = 96, y = 96, expected = "compute_to_draw" }, { kind = "point", x = 160, y = 96, expected = "offscreen_sampled" }, }, }, { id = "backend_features", example = "parity_backend_features", ledger = "bp-validation", negative_control = false, required_backends = { "vulkan", "opengl", "d3d11", "d3d12" }, assertions = { { kind = "point", x = 64, y = 64, expected = "mrt" }, { kind = "point", x = 128, y = 64, expected = "indirect_instance" }, { kind = "point", x = 192, y = 64, expected = "offscreen" }, }, }, } local function split_csv(value, defaults) if not value or value == true or value == "" then return defaults end local out = {} for item in tostring(value):gmatch("[^,]+") do local trimmed = item:gsub("^%s+", ""):gsub("%s+$", "") if trimmed ~= "" then out[#out + 1] = trimmed end end return out end local function parse_options(argv, start) local opts = { ["_"] = {} } local i = start or 1 while i <= #argv do local a = argv[i] if a:sub(1, 2) == "--" then local key, value = a:match("^%-%-([^=]+)=(.*)$") if not key then key = a:sub(3) local nextv = argv[i + 1] if nextv and nextv:sub(1, 2) ~= "--" then value = nextv i = i + 1 else value = true end end opts[key:gsub("-", "_")] = value else opts._[#opts._ + 1] = a end i = i + 1 end return opts end local function scene_by_id(id) for _, scene in ipairs(SCENES) do if scene.id == id or scene.example == id then return scene end end return nil end local function selected_scenes(value) local selected = split_csv(value, nil) if not selected then return SCENES end local out = {} for _, id in ipairs(selected) do local scene = scene_by_id(id) if not scene then error("unknown visual matrix scene: " .. tostring(id)) end out[#out + 1] = scene end return out end local function runtime_backend_supported(backend) if backend == "vulkan" then return true end if backend == "opengl" then return path.is_windows or process.uname() == "Linux" end if (backend == "d3d11" or backend == "d3d12") and path.is_windows then return true end return false end local function example_exists(root, name) return path.exists(path.join(root, "examples", name, "main.odin")) end local function result_dir(root, backend, scene_id) return path.join(root, ".build", "visual-matrix", backend, scene_id) end local function row_base(root, scene, backend, opts) local dir = result_dir(root, backend, scene.id) return { scene = scene.id, example = scene.example, ledger = scene.ledger, backend = backend, platform = process.uname(), status = "fail", skip_reason = nil, profile = path.join(dir, "profile.json"), capture = path.join(dir, "capture.png"), readback = path.join(dir, "readback.ppm"), negative_readback = path.join(dir, "readback-negative.ppm"), assertion_spec = path.join(dir, "assertions.json"), analysis = path.join(dir, "analysis.json"), negative_analysis = path.join(dir, "analysis-negative.json"), raw_result = path.join(dir, "run.json"), negative_raw_result = path.join(dir, "run-negative.json"), assertions = scene.assertions, negative_control_required = scene.negative_control, required_backends = scene.required_backends, dry_run = opts.dry_run and true or false, } end local function assertion_spec(scene) return { source_kind = "readback", width = scene.width or 256, height = scene.height or 256, origin = "top-left", assertions = scene.assertions, } end local function run_assertion(root, image, spec_path, analysis) local assert_cmd = table.concat({ "python3", path.shell_quote(path.join(root, "tools", "visual_assert.py")), path.shell_quote(image), "--spec", path.shell_quote(spec_path), "--json", path.shell_quote(analysis), }, " ") return process.run(process.with_cwd(root, assert_cmd)) end local function run_scene_mode(root, scene, backend, opts, row, mode) local dir = path.dirname(row.raw_result) local readback = row.readback local raw_result = row.raw_result local analysis = row.analysis if mode == "negative" then readback = row.negative_readback raw_result = row.negative_raw_result analysis = row.negative_analysis end path.remove_file(readback) path.remove_file(raw_result) path.remove_file(analysis) local frames = opts.frames or "120" local cmd_parts = { "lua tools/test.lua visual", path.shell_quote(scene.example), "--backend", path.shell_quote(backend), "--frames", path.shell_quote(frames), "--json", path.shell_quote(raw_result), "--no-screenshot", "--proof-output-dir", path.shell_quote(dir), "--proof-readback", path.shell_quote(readback), "--proof-mode", path.shell_quote(mode), "--proof-backend", path.shell_quote(backend), "--proof-scene", path.shell_quote(scene.id), } if process.uname() == "Linux" and (backend == "vulkan" or backend == "opengl") then cmd_parts[#cmd_parts + 1] = "--proof-headless" cmd_parts[#cmd_parts + 1] = "1" end local cmd = table.concat(cmd_parts, " ") local status = process.run(process.with_cwd(root, cmd)) local result = { mode = mode, status = "fail", readback = readback, raw_result = raw_result, analysis = analysis, run_exit_code = status, } if status ~= 0 then result.error = "visual run failed" return result end if not path.exists(readback) then result.error = "trusted GPU readback proof artifact missing" return result end local assert_status = run_assertion(root, readback, row.assertion_spec, analysis) result.assert_exit_code = assert_status if assert_status ~= 0 then result.error = "visual assertions failed" return result end result.status = "pass" return result end local function run_row(root, scene, backend, opts) local row = row_base(root, scene, backend, opts) if opts.dry_run then row.status = "skip" row.skip_reason = "dry-run" if scene.negative_control then row.negative_control = { mode = "negative", status = "skip", skip_reason = "dry-run" } end return row end if not runtime_backend_supported(backend) then row.status = "skip" row.skip_reason = "backend not runnable on this platform" if scene.negative_control then row.negative_control = { mode = "negative", status = "skip", skip_reason = row.skip_reason } end return row end if not example_exists(root, scene.example) then row.status = "fail" row.error = "example missing: " .. scene.example return row end path.mkdir_p(path.dirname(row.raw_result)) path.write_file(row.assertion_spec, json.encode(assertion_spec(scene)) .. "\n") local positive = run_scene_mode(root, scene, backend, opts, row, "positive") row.positive = positive row.status = positive.status row.error = positive.error row.exit_code = positive.assert_exit_code or positive.run_exit_code if row.status ~= "pass" then return row end if scene.negative_control then local negative = run_scene_mode(root, scene, backend, opts, row, "negative") row.negative_control = negative if negative.run_exit_code ~= 0 then row.status = "fail" row.error = "negative control run failed" return row end if negative.error == "trusted GPU readback proof artifact missing" then row.status = "fail" row.error = "negative control readback missing" return row end if negative.assert_exit_code ~= nil and negative.assert_exit_code ~= 0 then negative.status = "pass" negative.expected_failure = true negative.error = nil row.status = "pass" row.error = nil return row end row.status = "fail" row.error = "negative control unexpectedly passed assertions" negative.status = "fail" negative.error = row.error end return row end local function summarize(rows) local summary = { pass = 0, fail = 0, skip = 0, total = #rows } for _, row in ipairs(rows) do if row.status == "pass" then summary.pass = summary.pass + 1 elseif row.status == "skip" then summary.skip = summary.skip + 1 else summary.fail = summary.fail + 1 end end return summary end local function backend_required(scene, backend) for _, required in ipairs(scene.required_backends or {}) do if required == backend then return true end end return false end local function ledger_gate(scene, rows) local gate = { ledger = scene.ledger, scene = scene.id, required_backends = scene.required_backends or {}, status = "incomplete", pass = {}, fail = {}, skip = {}, missing = {}, } for _, backend in ipairs(gate.required_backends) do local found = false for _, row in ipairs(rows) do if row.scene == scene.id and row.backend == backend then found = true local negative_ok = true if scene.negative_control then negative_ok = row.negative_control and row.negative_control.status == "pass" and row.negative_control.expected_failure == true end if row.status == "pass" and negative_ok then gate.pass[#gate.pass + 1] = backend elseif row.status == "skip" then gate.skip[#gate.skip + 1] = backend else gate.fail[#gate.fail + 1] = backend end break end end if not found then gate.missing[#gate.missing + 1] = backend end end if #gate.pass == #gate.required_backends and #gate.fail == 0 and #gate.skip == 0 and #gate.missing == 0 then gate.status = "pass" end return gate end local function ledger_gates(rows) local gates = {} for _, scene in ipairs(SCENES) do gates[#gates + 1] = ledger_gate(scene, rows) end return gates end function M.run(opts, config) config = config or {} local root = config.root or path.repo_root("tools/test.lua") local rows = {} for _, scene in ipairs(selected_scenes(opts.scene)) do for _, backend in ipairs(split_csv(opts.backend, BACKENDS)) do rows[#rows + 1] = run_row(root, scene, backend, opts) end end local result = { kind = "gpu-visual-matrix", version = 1, rows = rows, summary = summarize(rows), ledger_gates = ledger_gates(rows), } if opts.json then path.mkdir_p(path.dirname(opts.json)) path.write_file(opts.json, json.encode(result) .. "\n") end return result end function M.selftest(root) local result = M.run({ dry_run = true, backend = "vulkan,d3d11", scene = "stencil_clip", }, { root = root or path.repo_root("tools/test.lua") }) assert(result.kind == "gpu-visual-matrix") assert(result.version == 1) assert(#result.rows == 2) assert(result.summary.skip == 2) assert(result.summary.pass == 0) assert(result.summary.fail == 0) assert(result.ledger_gates[1].status == "incomplete") assert(#result.ledger_gates[1].skip == 2) assert(#result.ledger_gates[1].missing == 2) assert(result.rows[1].status == "skip") assert(result.rows[1].skip_reason == "dry-run") assert(result.rows[1].negative_control_required == true) assert(result.rows[1].negative_control.status == "skip") local neg_scene = { id = "neg_gate", ledger = "bp-test", required_backends = { "vulkan" }, negative_control = true, } local missing_negative = ledger_gate(neg_scene, { { scene = "neg_gate", backend = "vulkan", status = "pass" }, }) assert(missing_negative.status == "incomplete") assert(#missing_negative.fail == 1) local unexpected_negative_pass = ledger_gate(neg_scene, { { scene = "neg_gate", backend = "vulkan", status = "pass", negative_control = { status = "fail", expected_failure = false }, }, }) assert(unexpected_negative_pass.status == "incomplete") assert(#unexpected_negative_pass.fail == 1) local expected_negative_failure = ledger_gate(neg_scene, { { scene = "neg_gate", backend = "vulkan", status = "pass", negative_control = { status = "pass", expected_failure = true }, }, }) assert(expected_negative_failure.status == "pass") end function M.main(argv, config) local opts = parse_options(argv, 2) local result = M.run(opts, config) print(json.encode(result)) return result.summary.fail == 0 and 0 or 1 end if arg and arg[0] and arg[0]:match("visual_matrix%.lua$") then os.exit(M.main(arg)) end return M