sharkk/gpu public

Changelog a309de6074a8

add visual parity matrix dry run

@sky · 1 month ago · parent b6ffe6b99fbc

2 added 2 modified 0 deleted

.build/visual-matrix/dry-run.json +1 -0 added

scripts/check.sh +1 -0 modified

…		10 unchanged lines hidden
11	11		if [ "${GPU_SKIP_LUA_SELFTEST:-0}" != "1" ]; then
12	12		echo "==> test: lua tooling"
13	13		lua tools/test.lua selftest
	14	+	lua tools/test.lua visual-matrix --dry-run --json .build/visual-matrix/dry-run.json >/dev/null
14	15		fi
15	16
16	17		echo "==> test: shader"
…		36 unchanged lines hidden

tools/test.lua +6 -0 modified

…		22 unchanged lines hidden
23	23		lua tools/test.lua check
24	24		lua tools/test.lua <example> [--backend NAME] [--seconds N] [--frames N] [--json PATH] [--profile PATH] [--visual] [--screenshot PATH] [--dry-run]
25	25		lua tools/test.lua visual <example> [--backend NAME] [--frames N] [--compare PATH]
	26	+	lua tools/test.lua visual-matrix [--backend LIST] [--scene LIST] [--frames N] [--json PATH] [--dry-run]
26	27		lua tools/test.lua bench <example\|all> [--backend NAME] [--seconds N] [--frames N] [--json PATH]
27	28		]])
28	29		end
…		285 unchanged lines hidden
314	315		end
315	316
316	317		local function selftest()
	318	+	local visual_matrix = require("tools.visual_matrix")
317	319		assert(json.encode({ b = 2, a = "x" }) == '{"a":"x","b":2}')
318	320		assert(path.basename(path.join("examples", "hello_window")) == "hello_window")
319	321		assert(example_exists("hello_window"))
…		8 unchanged lines hidden
328	330		assert(process.parse_elapsed_seconds("0:00.15") == 0.15)
329	331		assert(process.parse_elapsed_seconds("1:02:03") == 3723)
330	332		assert(process.command_exists("python3"))
	333	+	visual_matrix.selftest(root)
331	334		print("lua selftest ok")
332	335		end
333	336
…		49 unchanged lines hidden
383	386		print(json.encode(result))
384	387		if result.failed then return 1 end
385	388		return result.exit_code or 0
	389	+	elseif cmd == "visual-matrix" then
	390	+	local visual_matrix = require("tools.visual_matrix")
	391	+	return visual_matrix.main(argv, { root = root, window_collection = window_collection })
386	392		elseif cmd == "bench" then
387	393		cmd_bench(argv)
388	394		return 0
…		17 unchanged lines hidden

tools/visual_matrix.lua +251 -0 added

1	+	#!/usr/bin/env lua
2	+
3	+	package.path = table.concat({
4	+	"?.lua",
5	+	"?/init.lua",
6	+	"tools/lua/?.lua",
7	+	"tools/lua/?/init.lua",
8	+	package.path,
9	+	}, ";")
10	+
11	+	local json = require("tools.lua.json")
12	+	local path = require("tools.lua.path")
13	+	local process = require("tools.lua.process")
14	+
15	+	local M = {}
16	+
17	+	local BACKENDS = { "vulkan", "opengl", "d3d11", "d3d12" }
18	+
19	+	local SCENES = {
20	+	{
21	+	id = "stencil_clip",
22	+	example = "parity_stencil_clip",
23	+	ledger = "bp-stencil",
24	+	negative_control = true,
25	+	assertions = {
26	+	{ kind = "point", x = 48, y = 48, expected = "background" },
27	+	{ kind = "point", x = 128, y = 128, expected = "foreground" },
28	+	{ kind = "rect", x = 0, y = 0, w = 32, h = 256, expected = "background" },
29	+	},
30	+	},
31	+	{
32	+	id = "resource_sync",
33	+	example = "parity_resource_sync",
34	+	ledger = "bp-resource-sync",
35	+	negative_control = false,
36	+	assertions = {
37	+	{ kind = "point", x = 96, y = 96, expected = "compute_to_draw" },
38	+	{ kind = "point", x = 160, y = 96, expected = "offscreen_sampled" },
39	+	},
40	+	},
41	+	{
42	+	id = "backend_features",
43	+	example = "parity_backend_features",
44	+	ledger = "bp-validation",
45	+	negative_control = false,
46	+	assertions = {
47	+	{ kind = "point", x = 64, y = 64, expected = "mrt" },
48	+	{ kind = "point", x = 128, y = 64, expected = "indirect_instance" },
49	+	{ kind = "point", x = 192, y = 64, expected = "offscreen" },
50	+	},
51	+	},
52	+	}
53	+
54	+	local function split_csv(value, defaults)
55	+	if not value or value == true or value == "" then
56	+	return defaults
57	+	end
58	+	local out = {}
59	+	for item in tostring(value):gmatch("[^,]+") do
60	+	local trimmed = item:gsub("^%s+", ""):gsub("%s+$", "")
61	+	if trimmed ~= "" then out[#out + 1] = trimmed end
62	+	end
63	+	return out
64	+	end
65	+
66	+	local function parse_options(argv, start)
67	+	local opts = { ["_"] = {} }
68	+	local i = start or 1
69	+	while i <= #argv do
70	+	local a = argv[i]
71	+	if a:sub(1, 2) == "--" then
72	+	local key, value = a:match("^%-%-([^=]+)=(.*)$")
73	+	if not key then
74	+	key = a:sub(3)
75	+	local nextv = argv[i + 1]
76	+	if nextv and nextv:sub(1, 2) ~= "--" then
77	+	value = nextv
78	+	i = i + 1
79	+	else
80	+	value = true
81	+	end
82	+	end
83	+	opts[key:gsub("-", "_")] = value
84	+	else
85	+	opts._[#opts._ + 1] = a
86	+	end
87	+	i = i + 1
88	+	end
89	+	return opts
90	+	end
91	+
92	+	local function scene_by_id(id)
93	+	for _, scene in ipairs(SCENES) do
94	+	if scene.id == id or scene.example == id then return scene end
95	+	end
96	+	return nil
97	+	end
98	+
99	+	local function selected_scenes(value)
100	+	local selected = split_csv(value, nil)
101	+	if not selected then return SCENES end
102	+	local out = {}
103	+	for _, id in ipairs(selected) do
104	+	local scene = scene_by_id(id)
105	+	if not scene then error("unknown visual matrix scene: " .. tostring(id)) end
106	+	out[#out + 1] = scene
107	+	end
108	+	return out
109	+	end
110	+
111	+	local function runtime_backend_supported(backend)
112	+	if backend == "vulkan" then return true end
113	+	if backend == "opengl" then return path.is_windows or process.uname() == "Linux" end
114	+	if (backend == "d3d11" or backend == "d3d12") and path.is_windows then return true end
115	+	return false
116	+	end
117	+
118	+	local function example_exists(root, name)
119	+	return path.exists(path.join(root, "examples", name, "main.odin"))
120	+	end
121	+
122	+	local function result_dir(root, backend, scene_id)
123	+	return path.join(root, ".build", "visual-matrix", backend, scene_id)
124	+	end
125	+
126	+	local function row_base(root, scene, backend, opts)
127	+	local dir = result_dir(root, backend, scene.id)
128	+	return {
129	+	scene = scene.id,
130	+	example = scene.example,
131	+	ledger = scene.ledger,
132	+	backend = backend,
133	+	platform = process.uname(),
134	+	status = "fail",
135	+	skip_reason = nil,
136	+	profile = path.join(dir, "profile.json"),
137	+	capture = path.join(dir, "capture.png"),
138	+	analysis = path.join(dir, "analysis.json"),
139	+	raw_result = path.join(dir, "run.json"),
140	+	assertions = scene.assertions,
141	+	negative_control_required = scene.negative_control,
142	+	dry_run = opts.dry_run and true or false,
143	+	}
144	+	end
145	+
146	+	local function run_row(root, scene, backend, opts)
147	+	local row = row_base(root, scene, backend, opts)
148	+	if opts.dry_run then
149	+	row.status = "skip"
150	+	row.skip_reason = "dry-run"
151	+	return row
152	+	end
153	+	if not runtime_backend_supported(backend) then
154	+	row.status = "skip"
155	+	row.skip_reason = "backend not runnable on this platform"
156	+	return row
157	+	end
158	+	if not example_exists(root, scene.example) then
159	+	row.status = "fail"
160	+	row.error = "example missing: " .. scene.example
161	+	return row
162	+	end
163	+
164	+	path.mkdir_p(path.dirname(row.raw_result))
165	+	local frames = opts.frames or "120"
166	+	local cmd = table.concat({
167	+	"lua tools/test.lua visual",
168	+	path.shell_quote(scene.example),
169	+	"--backend", path.shell_quote(backend),
170	+	"--frames", path.shell_quote(frames),
171	+	"--json", path.shell_quote(row.raw_result),
172	+	"--screenshot", path.shell_quote(row.capture),
173	+	}, " ")
174	+	local status = process.run(process.with_cwd(root, cmd))
175	+	if status ~= 0 then
176	+	row.status = "fail"
177	+	row.error = "visual run failed"
178	+	row.exit_code = status
179	+	return row
180	+	end
181	+
182	+	row.status = "fail"
183	+	row.error = "exact assertion evaluator/readback proof not implemented yet"
184	+	return row
185	+	end
186	+
187	+	local function summarize(rows)
188	+	local summary = { pass = 0, fail = 0, skip = 0, total = #rows }
189	+	for _, row in ipairs(rows) do
190	+	if row.status == "pass" then
191	+	summary.pass = summary.pass + 1
192	+	elseif row.status == "skip" then
193	+	summary.skip = summary.skip + 1
194	+	else
195	+	summary.fail = summary.fail + 1
196	+	end
197	+	end
198	+	return summary
199	+	end
200	+
201	+	function M.run(opts, config)
202	+	config = config or {}
203	+	local root = config.root or path.repo_root("tools/test.lua")
204	+	local rows = {}
205	+	for _, scene in ipairs(selected_scenes(opts.scene)) do
206	+	for _, backend in ipairs(split_csv(opts.backend, BACKENDS)) do
207	+	rows[#rows + 1] = run_row(root, scene, backend, opts)
208	+	end
209	+	end
210	+	local result = {
211	+	kind = "gpu-visual-matrix",
212	+	version = 1,
213	+	rows = rows,
214	+	summary = summarize(rows),
215	+	}
216	+	if opts.json then
217	+	path.mkdir_p(path.dirname(opts.json))
218	+	path.write_file(opts.json, json.encode(result) .. "\n")
219	+	end
220	+	return result
221	+	end
222	+
223	+	function M.selftest(root)
224	+	local result = M.run({
225	+	dry_run = true,
226	+	backend = "vulkan,d3d11",
227	+	scene = "stencil_clip",
228	+	}, { root = root or path.repo_root("tools/test.lua") })
229	+	assert(result.kind == "gpu-visual-matrix")
230	+	assert(result.version == 1)
231	+	assert(#result.rows == 2)
232	+	assert(result.summary.skip == 2)
233	+	assert(result.summary.pass == 0)
234	+	assert(result.summary.fail == 0)
235	+	assert(result.rows[1].status == "skip")
236	+	assert(result.rows[1].skip_reason == "dry-run")
237	+	assert(result.rows[1].negative_control_required == true)
238	+	end
239	+
240	+	function M.main(argv, config)
241	+	local opts = parse_options(argv, 2)
242	+	local result = M.run(opts, config)
243	+	print(json.encode(result))
244	+	return result.summary.fail == 0 and 0 or 1
245	+	end
246	+
247	+	if arg and arg[0] and arg[0]:match("visual_matrix%.lua$") then
248	+	os.exit(M.main(arg))
249	+	end
250	+
251	+	return M