Harbor

Changelog 837ba96ab929

analyze visual matrix readbacks

@sky · 1 month ago · parent 022d952b0469
1 added 1 modified 0 deleted
.build/visual-matrix/dry-run.json +1 -0 added
1 + {"kind":"gpu-visual-matrix","ledger_gates":[{"fail":[],"ledger":"bp-stencil","missing":[],"pass":[],"required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"stencil_clip","skip":["vulkan","opengl","d3d11","d3d12"],"status":"incomplete"},{"fail":[],"ledger":"bp-resource-sync","missing":[],"pass":[],"required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"resource_sync","skip":["vulkan","opengl","d3d11","d3d12"],"status":"incomplete"},{"fail":[],"ledger":"bp-validation","missing":[],"pass":[],"required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"backend_features","skip":["vulkan","opengl","d3d11","d3d12"],"status":"incomplete"}],"rows":[{"analysis":"./.build/visual-matrix/vulkan/stencil_clip/analysis.json","assertion_spec":"./.build/visual-matrix/vulkan/stencil_clip/assertions.json","assertions":[{"expected":"background","kind":"point","x":48,"y":48},{"expected":"foreground","kind":"point","x":128,"y":128},{"expected":"background","h":256,"kind":"rect","w":32,"x":0,"y":0}],"backend":"vulkan","capture":"./.build/visual-matrix/vulkan/stencil_clip/capture.png","dry_run":true,"example":"parity_stencil_clip","ledger":"bp-stencil","negative_control_required":true,"platform":"Linux","profile":"./.build/visual-matrix/vulkan/stencil_clip/profile.json","raw_result":"./.build/visual-matrix/vulkan/stencil_clip/run.json","readback":"./.build/visual-matrix/vulkan/stencil_clip/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"stencil_clip","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/opengl/stencil_clip/analysis.json","assertion_spec":"./.build/visual-matrix/opengl/stencil_clip/assertions.json","assertions":[{"expected":"background","kind":"point","x":48,"y":48},{"expected":"foreground","kind":"point","x":128,"y":128},{"expected":"background","h":256,"kind":"rect","w":32,"x":0,"y":0}],"backend":"opengl","capture":"./.build/visual-matrix/opengl/stencil_clip/capture.png","dry_run":true,"example":"parity_stencil_clip","ledger":"bp-stencil","negative_control_required":true,"platform":"Linux","profile":"./.build/visual-matrix/opengl/stencil_clip/profile.json","raw_result":"./.build/visual-matrix/opengl/stencil_clip/run.json","readback":"./.build/visual-matrix/opengl/stencil_clip/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"stencil_clip","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/d3d11/stencil_clip/analysis.json","assertion_spec":"./.build/visual-matrix/d3d11/stencil_clip/assertions.json","assertions":[{"expected":"background","kind":"point","x":48,"y":48},{"expected":"foreground","kind":"point","x":128,"y":128},{"expected":"background","h":256,"kind":"rect","w":32,"x":0,"y":0}],"backend":"d3d11","capture":"./.build/visual-matrix/d3d11/stencil_clip/capture.png","dry_run":true,"example":"parity_stencil_clip","ledger":"bp-stencil","negative_control_required":true,"platform":"Linux","profile":"./.build/visual-matrix/d3d11/stencil_clip/profile.json","raw_result":"./.build/visual-matrix/d3d11/stencil_clip/run.json","readback":"./.build/visual-matrix/d3d11/stencil_clip/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"stencil_clip","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/d3d12/stencil_clip/analysis.json","assertion_spec":"./.build/visual-matrix/d3d12/stencil_clip/assertions.json","assertions":[{"expected":"background","kind":"point","x":48,"y":48},{"expected":"foreground","kind":"point","x":128,"y":128},{"expected":"background","h":256,"kind":"rect","w":32,"x":0,"y":0}],"backend":"d3d12","capture":"./.build/visual-matrix/d3d12/stencil_clip/capture.png","dry_run":true,"example":"parity_stencil_clip","ledger":"bp-stencil","negative_control_required":true,"platform":"Linux","profile":"./.build/visual-matrix/d3d12/stencil_clip/profile.json","raw_result":"./.build/visual-matrix/d3d12/stencil_clip/run.json","readback":"./.build/visual-matrix/d3d12/stencil_clip/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"stencil_clip","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/vulkan/resource_sync/analysis.json","assertion_spec":"./.build/visual-matrix/vulkan/resource_sync/assertions.json","assertions":[{"expected":"compute_to_draw","kind":"point","x":96,"y":96},{"expected":"offscreen_sampled","kind":"point","x":160,"y":96}],"backend":"vulkan","capture":"./.build/visual-matrix/vulkan/resource_sync/capture.png","dry_run":true,"example":"parity_resource_sync","ledger":"bp-resource-sync","negative_control_required":false,"platform":"Linux","profile":"./.build/visual-matrix/vulkan/resource_sync/profile.json","raw_result":"./.build/visual-matrix/vulkan/resource_sync/run.json","readback":"./.build/visual-matrix/vulkan/resource_sync/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"resource_sync","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/opengl/resource_sync/analysis.json","assertion_spec":"./.build/visual-matrix/opengl/resource_sync/assertions.json","assertions":[{"expected":"compute_to_draw","kind":"point","x":96,"y":96},{"expected":"offscreen_sampled","kind":"point","x":160,"y":96}],"backend":"opengl","capture":"./.build/visual-matrix/opengl/resource_sync/capture.png","dry_run":true,"example":"parity_resource_sync","ledger":"bp-resource-sync","negative_control_required":false,"platform":"Linux","profile":"./.build/visual-matrix/opengl/resource_sync/profile.json","raw_result":"./.build/visual-matrix/opengl/resource_sync/run.json","readback":"./.build/visual-matrix/opengl/resource_sync/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"resource_sync","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/d3d11/resource_sync/analysis.json","assertion_spec":"./.build/visual-matrix/d3d11/resource_sync/assertions.json","assertions":[{"expected":"compute_to_draw","kind":"point","x":96,"y":96},{"expected":"offscreen_sampled","kind":"point","x":160,"y":96}],"backend":"d3d11","capture":"./.build/visual-matrix/d3d11/resource_sync/capture.png","dry_run":true,"example":"parity_resource_sync","ledger":"bp-resource-sync","negative_control_required":false,"platform":"Linux","profile":"./.build/visual-matrix/d3d11/resource_sync/profile.json","raw_result":"./.build/visual-matrix/d3d11/resource_sync/run.json","readback":"./.build/visual-matrix/d3d11/resource_sync/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"resource_sync","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/d3d12/resource_sync/analysis.json","assertion_spec":"./.build/visual-matrix/d3d12/resource_sync/assertions.json","assertions":[{"expected":"compute_to_draw","kind":"point","x":96,"y":96},{"expected":"offscreen_sampled","kind":"point","x":160,"y":96}],"backend":"d3d12","capture":"./.build/visual-matrix/d3d12/resource_sync/capture.png","dry_run":true,"example":"parity_resource_sync","ledger":"bp-resource-sync","negative_control_required":false,"platform":"Linux","profile":"./.build/visual-matrix/d3d12/resource_sync/profile.json","raw_result":"./.build/visual-matrix/d3d12/resource_sync/run.json","readback":"./.build/visual-matrix/d3d12/resource_sync/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"resource_sync","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/vulkan/backend_features/analysis.json","assertion_spec":"./.build/visual-matrix/vulkan/backend_features/assertions.json","assertions":[{"expected":"mrt","kind":"point","x":64,"y":64},{"expected":"indirect_instance","kind":"point","x":128,"y":64},{"expected":"offscreen","kind":"point","x":192,"y":64}],"backend":"vulkan","capture":"./.build/visual-matrix/vulkan/backend_features/capture.png","dry_run":true,"example":"parity_backend_features","ledger":"bp-validation","negative_control_required":false,"platform":"Linux","profile":"./.build/visual-matrix/vulkan/backend_features/profile.json","raw_result":"./.build/visual-matrix/vulkan/backend_features/run.json","readback":"./.build/visual-matrix/vulkan/backend_features/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"backend_features","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/opengl/backend_features/analysis.json","assertion_spec":"./.build/visual-matrix/opengl/backend_features/assertions.json","assertions":[{"expected":"mrt","kind":"point","x":64,"y":64},{"expected":"indirect_instance","kind":"point","x":128,"y":64},{"expected":"offscreen","kind":"point","x":192,"y":64}],"backend":"opengl","capture":"./.build/visual-matrix/opengl/backend_features/capture.png","dry_run":true,"example":"parity_backend_features","ledger":"bp-validation","negative_control_required":false,"platform":"Linux","profile":"./.build/visual-matrix/opengl/backend_features/profile.json","raw_result":"./.build/visual-matrix/opengl/backend_features/run.json","readback":"./.build/visual-matrix/opengl/backend_features/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"backend_features","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/d3d11/backend_features/analysis.json","assertion_spec":"./.build/visual-matrix/d3d11/backend_features/assertions.json","assertions":[{"expected":"mrt","kind":"point","x":64,"y":64},{"expected":"indirect_instance","kind":"point","x":128,"y":64},{"expected":"offscreen","kind":"point","x":192,"y":64}],"backend":"d3d11","capture":"./.build/visual-matrix/d3d11/backend_features/capture.png","dry_run":true,"example":"parity_backend_features","ledger":"bp-validation","negative_control_required":false,"platform":"Linux","profile":"./.build/visual-matrix/d3d11/backend_features/profile.json","raw_result":"./.build/visual-matrix/d3d11/backend_features/run.json","readback":"./.build/visual-matrix/d3d11/backend_features/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"backend_features","skip_reason":"dry-run","status":"skip"},{"analysis":"./.build/visual-matrix/d3d12/backend_features/analysis.json","assertion_spec":"./.build/visual-matrix/d3d12/backend_features/assertions.json","assertions":[{"expected":"mrt","kind":"point","x":64,"y":64},{"expected":"indirect_instance","kind":"point","x":128,"y":64},{"expected":"offscreen","kind":"point","x":192,"y":64}],"backend":"d3d12","capture":"./.build/visual-matrix/d3d12/backend_features/capture.png","dry_run":true,"example":"parity_backend_features","ledger":"bp-validation","negative_control_required":false,"platform":"Linux","profile":"./.build/visual-matrix/d3d12/backend_features/profile.json","raw_result":"./.build/visual-matrix/d3d12/backend_features/run.json","readback":"./.build/visual-matrix/d3d12/backend_features/readback.ppm","required_backends":["vulkan","opengl","d3d11","d3d12"],"scene":"backend_features","skip_reason":"dry-run","status":"skip"}],"summary":{"fail":0,"pass":0,"skip":12,"total":12},"version":1}
tools/visual_matrix.lua +32 -2 modified
137 unchanged lines hidden
138 138 skip_reason = nil,
139 139 profile = path.join(dir, "profile.json"),
140 140 capture = path.join(dir, "capture.png"),
141 + readback = path.join(dir, "readback.ppm"),
142 + assertion_spec = path.join(dir, "assertions.json"),
141 143 analysis = path.join(dir, "analysis.json"),
142 144 raw_result = path.join(dir, "run.json"),
143 145 assertions = scene.assertions,
39 unchanged lines hidden
183 185 return row
184 186 end
185 187
186 - row.status = "fail"
187 - row.error = "trusted GPU readback proof artifact missing"
188 + if not path.exists(row.readback) then
189 + row.status = "fail"
190 + row.error = "trusted GPU readback proof artifact missing"
191 + return row
192 + end
193 +
194 + local spec = {
195 + source_kind = "readback",
196 + width = scene.width or 256,
197 + height = scene.height or 256,
198 + origin = "top-left",
199 + assertions = scene.assertions,
200 + }
201 + path.write_file(row.assertion_spec, json.encode(spec))
202 + local assert_cmd = table.concat({
203 + "python3",
204 + path.shell_quote(path.join(root, "tools", "visual_assert.py")),
205 + path.shell_quote(row.readback),
206 + "--spec", path.shell_quote(row.assertion_spec),
207 + "--json", path.shell_quote(row.analysis),
208 + }, " ")
209 + local assert_status = process.run(process.with_cwd(root, assert_cmd))
210 + if assert_status ~= 0 then
211 + row.status = "fail"
212 + row.error = "visual assertions failed"
213 + row.exit_code = assert_status
214 + return row
215 + end
216 + row.status = "pass"
217 + row.error = nil
188 218 return row
189 219 end
190 220
120 unchanged lines hidden