Harbor

sharkk/corvette public

Components 3 Issues Requests

Changelog e2ce381a48de

pin gpu proof contract

@sky · 1 month ago · parent 1faea7892cfb
0 added 2 modified 0 deleted
gpu/tools/test.lua +8 -3 modified
…20 unchanged lines hidden
21 21 lua tools/test.lua list
22 22 lua tools/test.lua selftest
23 23 lua tools/test.lua check
24 - lua tools/test.lua <example> [--backend NAME] [--seconds N] [--frames N] [--json PATH] [--profile PATH] [--visual] [--screenshot PATH] [--dry-run]
25 - lua tools/test.lua visual <example> [--backend NAME] [--frames N] [--compare PATH]
24 + lua tools/test.lua <example> [--backend NAME] [--seconds N] [--frames N] [--json PATH] [--profile PATH] [--visual] [--screenshot PATH] [--no-screenshot] [--dry-run]
25 + lua tools/test.lua visual <example> [--backend NAME] [--frames N] [--compare PATH] [--no-screenshot] [--proof-readback PATH] [--proof-mode MODE]
26 26 lua tools/test.lua visual-matrix [--backend LIST] [--scene LIST] [--frames N] [--json PATH] [--dry-run]
27 27 lua tools/test.lua bench <example|all> [--backend NAME] [--seconds N] [--frames N] [--json PATH]
28 28 ]])
…158 unchanged lines hidden
187 187
188 188 local profile = opts.profile or result_path("profiles", name, backend, "profile.json")
189 189 local metrics = result_path("metrics", name, backend, "time.txt")
190 - if opts.visual and not opts.screenshot then
190 + if opts.visual and not opts.screenshot and not opts.no_screenshot then
191 191 opts.screenshot = result_path("visual", name, backend, "png")
192 192 end
193 193 if opts.visual and not opts.frames and not opts.seconds then
…11 unchanged lines hidden
205 205 GPU_BACKEND = backend,
206 206 GPU_PROFILE_JSON = profile,
207 207 }
208 + if opts.proof_output_dir then env.GPU_PROOF_OUTPUT_DIR = opts.proof_output_dir end
209 + if opts.proof_readback then env.GPU_PROOF_READBACK = opts.proof_readback end
210 + if opts.proof_mode then env.GPU_PROOF_MODE = opts.proof_mode end
211 + if opts.proof_backend then env.GPU_PROOF_BACKEND = opts.proof_backend end
212 + if opts.proof_scene then env.GPU_PROOF_SCENE = opts.proof_scene end
208 213 if opts.seconds then env.GPU_TEST_SECONDS = opts.seconds end
209 214 if opts.frames then env.GPU_TEST_FRAMES = opts.frames end
210 215 if opts.screenshot then
…202 unchanged lines hidden
gpu/tools/visual_matrix.lua +156 -42 modified
…138 unchanged lines hidden
139 139 profile = path.join(dir, "profile.json"),
140 140 capture = path.join(dir, "capture.png"),
141 141 readback = path.join(dir, "readback.ppm"),
142 + negative_readback = path.join(dir, "readback-negative.ppm"),
142 143 assertion_spec = path.join(dir, "assertions.json"),
143 144 analysis = path.join(dir, "analysis.json"),
145 + negative_analysis = path.join(dir, "analysis-negative.json"),
144 146 raw_result = path.join(dir, "run.json"),
147 + negative_raw_result = path.join(dir, "run-negative.json"),
145 148 assertions = scene.assertions,
146 149 negative_control_required = scene.negative_control,
147 150 required_backends = scene.required_backends,
…1 unchanged lines hidden
149 152 }
150 153 end
151 154
155 + local function assertion_spec(scene)
156 + return {
157 + source_kind = "readback",
158 + width = scene.width or 256,
159 + height = scene.height or 256,
160 + origin = "top-left",
161 + assertions = scene.assertions,
162 + }
163 + end
164 +
165 + local function run_assertion(root, image, spec_path, analysis)
166 + local assert_cmd = table.concat({
167 + "python3",
168 + path.shell_quote(path.join(root, "tools", "visual_assert.py")),
169 + path.shell_quote(image),
170 + "--spec", path.shell_quote(spec_path),
171 + "--json", path.shell_quote(analysis),
172 + }, " ")
173 + return process.run(process.with_cwd(root, assert_cmd))
174 + end
175 +
176 + local function run_scene_mode(root, scene, backend, opts, row, mode)
177 + local dir = path.dirname(row.raw_result)
178 + local readback = row.readback
179 + local raw_result = row.raw_result
180 + local analysis = row.analysis
181 + if mode == "negative" then
182 + readback = row.negative_readback
183 + raw_result = row.negative_raw_result
184 + analysis = row.negative_analysis
185 + end
186 + path.remove_file(readback)
187 + path.remove_file(raw_result)
188 + path.remove_file(analysis)
189 + local frames = opts.frames or "120"
190 + local cmd = table.concat({
191 + "lua tools/test.lua visual",
192 + path.shell_quote(scene.example),
193 + "--backend", path.shell_quote(backend),
194 + "--frames", path.shell_quote(frames),
195 + "--json", path.shell_quote(raw_result),
196 + "--no-screenshot",
197 + "--proof-output-dir", path.shell_quote(dir),
198 + "--proof-readback", path.shell_quote(readback),
199 + "--proof-mode", path.shell_quote(mode),
200 + "--proof-backend", path.shell_quote(backend),
201 + "--proof-scene", path.shell_quote(scene.id),
202 + }, " ")
203 + local status = process.run(process.with_cwd(root, cmd))
204 + local result = {
205 + mode = mode,
206 + status = "fail",
207 + readback = readback,
208 + raw_result = raw_result,
209 + analysis = analysis,
210 + run_exit_code = status,
211 + }
212 + if status ~= 0 then
213 + result.error = "visual run failed"
214 + return result
215 + end
216 + if not path.exists(readback) then
217 + result.error = "trusted GPU readback proof artifact missing"
218 + return result
219 + end
220 + local assert_status = run_assertion(root, readback, row.assertion_spec, analysis)
221 + result.assert_exit_code = assert_status
222 + if assert_status ~= 0 then
223 + result.error = "visual assertions failed"
224 + return result
225 + end
226 + result.status = "pass"
227 + return result
228 + end
229 +
152 230 local function run_row(root, scene, backend, opts)
153 231 local row = row_base(root, scene, backend, opts)
154 232 if opts.dry_run then
155 233 row.status = "skip"
156 234 row.skip_reason = "dry-run"
235 + if scene.negative_control then
236 + row.negative_control = { mode = "negative", status = "skip", skip_reason = "dry-run" }
237 + end
157 238 return row
158 239 end
159 240 if not runtime_backend_supported(backend) then
160 241 row.status = "skip"
161 242 row.skip_reason = "backend not runnable on this platform"
243 + if scene.negative_control then
244 + row.negative_control = { mode = "negative", status = "skip", skip_reason = row.skip_reason }
245 + end
162 246 return row
163 247 end
164 248 if not example_exists(root, scene.example) then
…3 unchanged lines hidden
168 252 end
169 253
170 254 path.mkdir_p(path.dirname(row.raw_result))
171 - local frames = opts.frames or "120"
172 - local cmd = table.concat({
173 - "lua tools/test.lua visual",
174 - path.shell_quote(scene.example),
175 - "--backend", path.shell_quote(backend),
176 - "--frames", path.shell_quote(frames),
177 - "--json", path.shell_quote(row.raw_result),
178 - "--screenshot", path.shell_quote(row.capture),
179 - }, " ")
180 - local status = process.run(process.with_cwd(root, cmd))
181 - if status ~= 0 then
182 - row.status = "fail"
183 - row.error = "visual run failed"
184 - row.exit_code = status
185 - return row
186 - end
255 + path.write_file(row.assertion_spec, json.encode(assertion_spec(scene)) .. "\n")
187 256
188 - if not path.exists(row.readback) then
189 - row.status = "fail"
190 - row.error = "trusted GPU readback proof artifact missing"
257 + local positive = run_scene_mode(root, scene, backend, opts, row, "positive")
258 + row.positive = positive
259 + row.status = positive.status
260 + row.error = positive.error
261 + row.exit_code = positive.assert_exit_code or positive.run_exit_code
262 + if row.status ~= "pass" then
191 263 return row
192 264 end
193 265
194 - local spec = {
195 - source_kind = "readback",
196 - width = scene.width or 256,
197 - height = scene.height or 256,
198 - origin = "top-left",
199 - assertions = scene.assertions,
200 - }
201 - path.write_file(row.assertion_spec, json.encode(spec))
202 - local assert_cmd = table.concat({
203 - "python3",
204 - path.shell_quote(path.join(root, "tools", "visual_assert.py")),
205 - path.shell_quote(row.readback),
206 - "--spec", path.shell_quote(row.assertion_spec),
207 - "--json", path.shell_quote(row.analysis),
208 - }, " ")
209 - local assert_status = process.run(process.with_cwd(root, assert_cmd))
210 - if assert_status ~= 0 then
266 + if scene.negative_control then
267 + local negative = run_scene_mode(root, scene, backend, opts, row, "negative")
268 + row.negative_control = negative
269 + if negative.run_exit_code ~= 0 then
270 + row.status = "fail"
271 + row.error = "negative control run failed"
272 + return row
273 + end
274 + if negative.error == "trusted GPU readback proof artifact missing" then
275 + row.status = "fail"
276 + row.error = "negative control readback missing"
277 + return row
278 + end
279 + if negative.assert_exit_code ~= nil and negative.assert_exit_code ~= 0 then
280 + negative.status = "pass"
281 + negative.expected_failure = true
282 + negative.error = nil
283 + row.status = "pass"
284 + row.error = nil
285 + return row
286 + end
211 287 row.status = "fail"
212 - row.error = "visual assertions failed"
213 - row.exit_code = assert_status
214 - return row
288 + row.error = "negative control unexpectedly passed assertions"
289 + negative.status = "fail"
290 + negative.error = row.error
215 291 end
216 - row.status = "pass"
217 - row.error = nil
218 292 return row
219 293 end
220 294
…34 unchanged lines hidden
255 329 for _, row in ipairs(rows) do
256 330 if row.scene == scene.id and row.backend == backend then
257 331 found = true
258 - if row.status == "pass" then
332 + local negative_ok = true
333 + if scene.negative_control then
334 + negative_ok = row.negative_control and
335 + row.negative_control.status == "pass" and
336 + row.negative_control.expected_failure == true
337 + end
338 + if row.status == "pass" and negative_ok then
259 339 gate.pass[#gate.pass + 1] = backend
260 340 elseif row.status == "skip" then
261 341 gate.skip[#gate.skip + 1] = backend
…62 unchanged lines hidden
324 404 assert(result.rows[1].status == "skip")
325 405 assert(result.rows[1].skip_reason == "dry-run")
326 406 assert(result.rows[1].negative_control_required == true)
407 + assert(result.rows[1].negative_control.status == "skip")
408 +
409 + local neg_scene = {
410 + id = "neg_gate",
411 + ledger = "bp-test",
412 + required_backends = { "vulkan" },
413 + negative_control = true,
414 + }
415 + local missing_negative = ledger_gate(neg_scene, {
416 + { scene = "neg_gate", backend = "vulkan", status = "pass" },
417 + })
418 + assert(missing_negative.status == "incomplete")
419 + assert(#missing_negative.fail == 1)
420 +
421 + local unexpected_negative_pass = ledger_gate(neg_scene, {
422 + {
423 + scene = "neg_gate",
424 + backend = "vulkan",
425 + status = "pass",
426 + negative_control = { status = "fail", expected_failure = false },
427 + },
428 + })
429 + assert(unexpected_negative_pass.status == "incomplete")
430 + assert(#unexpected_negative_pass.fail == 1)
431 +
432 + local expected_negative_failure = ledger_gate(neg_scene, {
433 + {
434 + scene = "neg_gate",
435 + backend = "vulkan",
436 + status = "pass",
437 + negative_control = { status = "pass", expected_failure = true },
438 + },
439 + })
440 + assert(expected_negative_failure.status == "pass")
327 441 end
328 442
329 443 function M.main(argv, config)
…11 unchanged lines hidden