sharkk/corvette public

branch main
showing the latest snapshot on main
visual_matrix.lua 12.7 KB · Lua
#!/usr/bin/env lua

package.path = table.concat({
	"?.lua",
	"?/init.lua",
	"tools/lua/?.lua",
	"tools/lua/?/init.lua",
	package.path,
}, ";")

local json = require("tools.lua.json")
local path = require("tools.lua.path")
local process = require("tools.lua.process")

local M = {}

local BACKENDS = { "vulkan", "opengl", "d3d11", "d3d12" }

local SCENES = {
	{
		id = "stencil_clip",
		example = "parity_stencil_clip",
		ledger = "bp-stencil",
		negative_control = true,
		required_backends = { "vulkan", "opengl", "d3d11", "d3d12" },
		assertions = {
			{ kind = "point", x = 48, y = 48, expected = "background" },
			{ kind = "point", x = 128, y = 128, expected = "foreground" },
			{ kind = "rect", x = 0, y = 0, w = 32, h = 256, expected = "background" },
		},
	},
	{
		id = "resource_sync",
		example = "parity_resource_sync",
		ledger = "bp-resource-sync",
		negative_control = false,
		required_backends = { "vulkan", "opengl", "d3d11", "d3d12" },
		assertions = {
			{ kind = "point", x = 96, y = 96, expected = "compute_to_draw" },
			{ kind = "point", x = 160, y = 96, expected = "offscreen_sampled" },
		},
	},
	{
		id = "backend_features",
		example = "parity_backend_features",
		ledger = "bp-validation",
		negative_control = false,
		required_backends = { "vulkan", "opengl", "d3d11", "d3d12" },
		assertions = {
			{ kind = "point", x = 64, y = 64, expected = "mrt" },
			{ kind = "point", x = 128, y = 64, expected = "indirect_instance" },
			{ kind = "point", x = 192, y = 64, expected = "offscreen" },
		},
	},
}

local function split_csv(value, defaults)
	if not value or value == true or value == "" then
		return defaults
	end
	local out = {}
	for item in tostring(value):gmatch("[^,]+") do
		local trimmed = item:gsub("^%s+", ""):gsub("%s+$", "")
		if trimmed ~= "" then out[#out + 1] = trimmed end
	end
	return out
end

local function parse_options(argv, start)
	local opts = { ["_"] = {} }
	local i = start or 1
	while i <= #argv do
		local a = argv[i]
		if a:sub(1, 2) == "--" then
			local key, value = a:match("^%-%-([^=]+)=(.*)$")
			if not key then
				key = a:sub(3)
				local nextv = argv[i + 1]
				if nextv and nextv:sub(1, 2) ~= "--" then
					value = nextv
					i = i + 1
				else
					value = true
				end
			end
			opts[key:gsub("-", "_")] = value
		else
			opts._[#opts._ + 1] = a
		end
		i = i + 1
	end
	return opts
end

local function scene_by_id(id)
	for _, scene in ipairs(SCENES) do
		if scene.id == id or scene.example == id then return scene end
	end
	return nil
end

local function selected_scenes(value)
	local selected = split_csv(value, nil)
	if not selected then return SCENES end
	local out = {}
	for _, id in ipairs(selected) do
		local scene = scene_by_id(id)
		if not scene then error("unknown visual matrix scene: " .. tostring(id)) end
		out[#out + 1] = scene
	end
	return out
end

local function runtime_backend_supported(backend)
	if backend == "vulkan" then return true end
	if backend == "opengl" then return path.is_windows or process.uname() == "Linux" end
	if (backend == "d3d11" or backend == "d3d12") and path.is_windows then return true end
	return false
end

local function example_exists(root, name)
	return path.exists(path.join(root, "examples", name, "main.odin"))
end

local function result_dir(root, backend, scene_id)
	return path.join(root, ".build", "visual-matrix", backend, scene_id)
end

local function row_base(root, scene, backend, opts)
	local dir = result_dir(root, backend, scene.id)
	return {
		scene = scene.id,
		example = scene.example,
		ledger = scene.ledger,
		backend = backend,
		platform = process.uname(),
		status = "fail",
		skip_reason = nil,
		profile = path.join(dir, "profile.json"),
		capture = path.join(dir, "capture.png"),
		readback = path.join(dir, "readback.ppm"),
		negative_readback = path.join(dir, "readback-negative.ppm"),
		assertion_spec = path.join(dir, "assertions.json"),
		analysis = path.join(dir, "analysis.json"),
		negative_analysis = path.join(dir, "analysis-negative.json"),
		raw_result = path.join(dir, "run.json"),
		negative_raw_result = path.join(dir, "run-negative.json"),
		assertions = scene.assertions,
		negative_control_required = scene.negative_control,
		required_backends = scene.required_backends,
		dry_run = opts.dry_run and true or false,
	}
end

local function assertion_spec(scene)
	return {
		source_kind = "readback",
		width = scene.width or 256,
		height = scene.height or 256,
		origin = "top-left",
		assertions = scene.assertions,
	}
end

local function run_assertion(root, image, spec_path, analysis)
	local assert_cmd = table.concat({
		"python3",
		path.shell_quote(path.join(root, "tools", "visual_assert.py")),
		path.shell_quote(image),
		"--spec", path.shell_quote(spec_path),
		"--json", path.shell_quote(analysis),
	}, " ")
	return process.run(process.with_cwd(root, assert_cmd))
end

local function run_scene_mode(root, scene, backend, opts, row, mode)
	local dir = path.dirname(row.raw_result)
	local readback = row.readback
	local raw_result = row.raw_result
	local analysis = row.analysis
	if mode == "negative" then
		readback = row.negative_readback
		raw_result = row.negative_raw_result
		analysis = row.negative_analysis
	end
	path.remove_file(readback)
	path.remove_file(raw_result)
	path.remove_file(analysis)
	local frames = opts.frames or "120"
	local cmd_parts = {
		"lua tools/test.lua visual",
		path.shell_quote(scene.example),
		"--backend", path.shell_quote(backend),
		"--frames", path.shell_quote(frames),
		"--json", path.shell_quote(raw_result),
		"--no-screenshot",
		"--proof-output-dir", path.shell_quote(dir),
		"--proof-readback", path.shell_quote(readback),
		"--proof-mode", path.shell_quote(mode),
		"--proof-backend", path.shell_quote(backend),
		"--proof-scene", path.shell_quote(scene.id),
	}
	if process.uname() == "Linux" and (backend == "vulkan" or backend == "opengl") then
		cmd_parts[#cmd_parts + 1] = "--proof-headless"
		cmd_parts[#cmd_parts + 1] = "1"
	end
	local cmd = table.concat(cmd_parts, " ")
	local status = process.run(process.with_cwd(root, cmd))
	local result = {
		mode = mode,
		status = "fail",
		readback = readback,
		raw_result = raw_result,
		analysis = analysis,
		run_exit_code = status,
	}
	if status ~= 0 then
		result.error = "visual run failed"
		return result
	end
	if not path.exists(readback) then
		result.error = "trusted GPU readback proof artifact missing"
		return result
	end
	local assert_status = run_assertion(root, readback, row.assertion_spec, analysis)
	result.assert_exit_code = assert_status
	if assert_status ~= 0 then
		result.error = "visual assertions failed"
		return result
	end
	result.status = "pass"
	return result
end

local function run_row(root, scene, backend, opts)
	local row = row_base(root, scene, backend, opts)
	if opts.dry_run then
		row.status = "skip"
		row.skip_reason = "dry-run"
		if scene.negative_control then
			row.negative_control = { mode = "negative", status = "skip", skip_reason = "dry-run" }
		end
		return row
	end
	if not runtime_backend_supported(backend) then
		row.status = "skip"
		row.skip_reason = "backend not runnable on this platform"
		if scene.negative_control then
			row.negative_control = { mode = "negative", status = "skip", skip_reason = row.skip_reason }
		end
		return row
	end
	if not example_exists(root, scene.example) then
		row.status = "fail"
		row.error = "example missing: " .. scene.example
		return row
	end

	path.mkdir_p(path.dirname(row.raw_result))
	path.write_file(row.assertion_spec, json.encode(assertion_spec(scene)) .. "\n")

	local positive = run_scene_mode(root, scene, backend, opts, row, "positive")
	row.positive = positive
	row.status = positive.status
	row.error = positive.error
	row.exit_code = positive.assert_exit_code or positive.run_exit_code
	if row.status ~= "pass" then
		return row
	end

	if scene.negative_control then
		local negative = run_scene_mode(root, scene, backend, opts, row, "negative")
		row.negative_control = negative
		if negative.run_exit_code ~= 0 then
			row.status = "fail"
			row.error = "negative control run failed"
			return row
		end
		if negative.error == "trusted GPU readback proof artifact missing" then
			row.status = "fail"
			row.error = "negative control readback missing"
			return row
		end
		if negative.assert_exit_code ~= nil and negative.assert_exit_code ~= 0 then
			negative.status = "pass"
			negative.expected_failure = true
			negative.error = nil
			row.status = "pass"
			row.error = nil
			return row
		end
		row.status = "fail"
		row.error = "negative control unexpectedly passed assertions"
		negative.status = "fail"
		negative.error = row.error
	end
	return row
end

local function summarize(rows)
	local summary = { pass = 0, fail = 0, skip = 0, total = #rows }
	for _, row in ipairs(rows) do
		if row.status == "pass" then
			summary.pass = summary.pass + 1
		elseif row.status == "skip" then
			summary.skip = summary.skip + 1
		else
			summary.fail = summary.fail + 1
		end
	end
	return summary
end

local function backend_required(scene, backend)
	for _, required in ipairs(scene.required_backends or {}) do
		if required == backend then return true end
	end
	return false
end

local function ledger_gate(scene, rows)
	local gate = {
		ledger = scene.ledger,
		scene = scene.id,
		required_backends = scene.required_backends or {},
		status = "incomplete",
		pass = {},
		fail = {},
		skip = {},
		missing = {},
	}
	for _, backend in ipairs(gate.required_backends) do
		local found = false
		for _, row in ipairs(rows) do
			if row.scene == scene.id and row.backend == backend then
				found = true
				local negative_ok = true
				if scene.negative_control then
					negative_ok = row.negative_control and
						row.negative_control.status == "pass" and
						row.negative_control.expected_failure == true
				end
				if row.status == "pass" and negative_ok then
					gate.pass[#gate.pass + 1] = backend
				elseif row.status == "skip" then
					gate.skip[#gate.skip + 1] = backend
				else
					gate.fail[#gate.fail + 1] = backend
				end
				break
			end
		end
		if not found then
			gate.missing[#gate.missing + 1] = backend
		end
	end
	if #gate.pass == #gate.required_backends and #gate.fail == 0 and #gate.skip == 0 and #gate.missing == 0 then
		gate.status = "pass"
	end
	return gate
end

local function ledger_gates(rows)
	local gates = {}
	for _, scene in ipairs(SCENES) do
		gates[#gates + 1] = ledger_gate(scene, rows)
	end
	return gates
end

function M.run(opts, config)
	config = config or {}
	local root = config.root or path.repo_root("tools/test.lua")
	local rows = {}
	for _, scene in ipairs(selected_scenes(opts.scene)) do
		for _, backend in ipairs(split_csv(opts.backend, BACKENDS)) do
			rows[#rows + 1] = run_row(root, scene, backend, opts)
		end
	end
	local result = {
		kind = "gpu-visual-matrix",
		version = 1,
		rows = rows,
		summary = summarize(rows),
		ledger_gates = ledger_gates(rows),
	}
	if opts.json then
		path.mkdir_p(path.dirname(opts.json))
		path.write_file(opts.json, json.encode(result) .. "\n")
	end
	return result
end

function M.selftest(root)
	local result = M.run({
		dry_run = true,
		backend = "vulkan,d3d11",
		scene = "stencil_clip",
	}, { root = root or path.repo_root("tools/test.lua") })
	assert(result.kind == "gpu-visual-matrix")
	assert(result.version == 1)
	assert(#result.rows == 2)
	assert(result.summary.skip == 2)
	assert(result.summary.pass == 0)
	assert(result.summary.fail == 0)
	assert(result.ledger_gates[1].status == "incomplete")
	assert(#result.ledger_gates[1].skip == 2)
	assert(#result.ledger_gates[1].missing == 2)
	assert(result.rows[1].status == "skip")
	assert(result.rows[1].skip_reason == "dry-run")
	assert(result.rows[1].negative_control_required == true)
	assert(result.rows[1].negative_control.status == "skip")

	local neg_scene = {
		id = "neg_gate",
		ledger = "bp-test",
		required_backends = { "vulkan" },
		negative_control = true,
	}
	local missing_negative = ledger_gate(neg_scene, {
		{ scene = "neg_gate", backend = "vulkan", status = "pass" },
	})
	assert(missing_negative.status == "incomplete")
	assert(#missing_negative.fail == 1)

	local unexpected_negative_pass = ledger_gate(neg_scene, {
		{
			scene = "neg_gate",
			backend = "vulkan",
			status = "pass",
			negative_control = { status = "fail", expected_failure = false },
		},
	})
	assert(unexpected_negative_pass.status == "incomplete")
	assert(#unexpected_negative_pass.fail == 1)

	local expected_negative_failure = ledger_gate(neg_scene, {
		{
			scene = "neg_gate",
			backend = "vulkan",
			status = "pass",
			negative_control = { status = "pass", expected_failure = true },
		},
	})
	assert(expected_negative_failure.status == "pass")
end

function M.main(argv, config)
	local opts = parse_options(argv, 2)
	local result = M.run(opts, config)
	print(json.encode(result))
	return result.summary.fail == 0 and 0 or 1
end

if arg and arg[0] and arg[0]:match("visual_matrix%.lua$") then
	os.exit(M.main(arg))
end

return M