Harbor

sharkk/gpu public

Components Issues 1 Requests

Changelog c5254a455517

add exact visual assertions

@sky · 1 month ago · parent f33f3bd1c69e
1 added 1 modified 0 deleted
tools/test.lua +1 -0 modified
…329 unchanged lines hidden
330 330 assert(process.parse_elapsed_seconds("0:00.15") == 0.15)
331 331 assert(process.parse_elapsed_seconds("1:02:03") == 3723)
332 332 assert(process.command_exists("python3"))
333 + assert(process.run("python3 " .. path.shell_quote(path.join(root, "tools", "visual_assert.py")) .. " --selftest") == 0)
333 334 visual_matrix.selftest(root)
334 335 print("lua selftest ok")
335 336 end
…76 unchanged lines hidden
tools/visual_assert.py +279 -0 added
1 + #!/usr/bin/env python3
2 + import argparse
3 + import json
4 + import os
5 + import struct
6 + import subprocess
7 + import sys
8 + import tempfile
9 +
10 +
11 + PALETTE = {
12 + "background": [16, 16, 24, 255],
13 + "foreground": [240, 48, 96, 255],
14 + "compute_to_draw": [48, 224, 128, 255],
15 + "offscreen_sampled": [64, 128, 240, 255],
16 + "mrt": [240, 64, 64, 255],
17 + "indirect_instance": [64, 240, 64, 255],
18 + "offscreen": [64, 64, 240, 255],
19 + }
20 +
21 +
22 + def _read_token(f):
23 + token = bytearray()
24 + while True:
25 + c = f.read(1)
26 + if not c:
27 + return None
28 + if c == b"#":
29 + f.readline()
30 + continue
31 + if not c.isspace():
32 + token.extend(c)
33 + break
34 + while True:
35 + c = f.read(1)
36 + if not c or c.isspace():
37 + break
38 + token.extend(c)
39 + return token.decode("ascii")
40 +
41 +
42 + def load_ppm(path):
43 + with open(path, "rb") as f:
44 + magic = _read_token(f)
45 + if magic != "P6":
46 + raise ValueError("unsupported PPM magic")
47 + width = int(_read_token(f))
48 + height = int(_read_token(f))
49 + max_value = int(_read_token(f))
50 + if max_value != 255:
51 + raise ValueError("unsupported PPM max value")
52 + data = f.read(width * height * 3)
53 + if len(data) != width * height * 3:
54 + raise ValueError("truncated PPM image")
55 + rgba = bytearray(width * height * 4)
56 + for i in range(width * height):
57 + rgba[i * 4:i * 4 + 3] = data[i * 3:i * 3 + 3]
58 + rgba[i * 4 + 3] = 255
59 + return width, height, bytes(rgba)
60 +
61 +
62 + def load_with_pillow(path):
63 + from PIL import Image
64 + img = Image.open(path).convert("RGBA")
65 + return img.width, img.height, img.tobytes()
66 +
67 +
68 + def load_with_magick(path):
69 + result = subprocess.run(
70 + ["magick", path, "rgba:-"],
71 + stdout=subprocess.PIPE,
72 + stderr=subprocess.PIPE,
73 + )
74 + if result.returncode != 0:
75 + raise ValueError(result.stderr.decode("utf-8", "replace").strip() or "magick failed")
76 + size = subprocess.run(
77 + ["magick", "identify", "-format", "%w %h", path],
78 + stdout=subprocess.PIPE,
79 + stderr=subprocess.PIPE,
80 + text=True,
81 + )
82 + if size.returncode != 0:
83 + raise ValueError(size.stderr.strip() or "magick identify failed")
84 + width, height = [int(v) for v in size.stdout.split()]
85 + return width, height, result.stdout
86 +
87 +
88 + def load_image(path):
89 + lower = path.lower()
90 + if lower.endswith(".ppm"):
91 + return load_ppm(path)
92 + try:
93 + return load_with_pillow(path)
94 + except Exception:
95 + return load_with_magick(path)
96 +
97 +
98 + def color_for(value):
99 + if isinstance(value, str):
100 + if value not in PALETTE:
101 + raise ValueError("unknown palette color: " + value)
102 + return PALETTE[value]
103 + if isinstance(value, list) and len(value) in (3, 4):
104 + out = [int(v) for v in value]
105 + if len(out) == 3:
106 + out.append(255)
107 + return out
108 + raise ValueError("invalid expected color")
109 +
110 +
111 + def pixel_at(width, data, x, y):
112 + idx = (y * width + x) * 4
113 + return list(data[idx:idx + 4])
114 +
115 +
116 + def within(actual, expected, tolerance):
117 + return all(abs(int(a) - int(e)) <= tolerance for a, e in zip(actual, expected))
118 +
119 +
120 + def evaluate_assertion(width, height, data, assertion):
121 + kind = assertion.get("kind")
122 + expected = color_for(assertion.get("expected"))
123 + tolerance = int(assertion.get("tolerance", 0))
124 + result = {
125 + "kind": kind,
126 + "expected": expected,
127 + "tolerance": tolerance,
128 + "pass": False,
129 + }
130 + if kind == "point":
131 + x = int(assertion["x"])
132 + y = int(assertion["y"])
133 + result["x"] = x
134 + result["y"] = y
135 + if x < 0 or y < 0 or x >= width or y >= height:
136 + result["reason"] = "point outside image"
137 + return result
138 + actual = pixel_at(width, data, x, y)
139 + result["actual"] = actual
140 + result["pass"] = within(actual, expected, tolerance)
141 + if not result["pass"]:
142 + result["reason"] = "point color mismatch"
143 + return result
144 + if kind == "rect":
145 + x = int(assertion["x"])
146 + y = int(assertion["y"])
147 + w = int(assertion["w"])
148 + h = int(assertion["h"])
149 + result.update({"x": x, "y": y, "w": w, "h": h})
150 + if x < 0 or y < 0 or w <= 0 or h <= 0 or x + w > width or y + h > height:
151 + result["reason"] = "rect outside image"
152 + return result
153 + mismatches = []
154 + for yy in range(y, y + h):
155 + for xx in range(x, x + w):
156 + actual = pixel_at(width, data, xx, yy)
157 + if not within(actual, expected, tolerance):
158 + mismatches.append({"x": xx, "y": yy, "actual": actual})
159 + if len(mismatches) >= 8:
160 + break
161 + if len(mismatches) >= 8:
162 + break
163 + result["mismatches"] = mismatches
164 + result["pass"] = len(mismatches) == 0
165 + if not result["pass"]:
166 + result["reason"] = "rect color mismatch"
167 + return result
168 + result["reason"] = "unknown assertion kind"
169 + return result
170 +
171 +
172 + def evaluate(path, spec):
173 + if spec.get("source_kind") == "screenshot":
174 + return {
175 + "ok": False,
176 + "path": path,
177 + "source_kind": "screenshot",
178 + "error": "screenshots are smoke artifacts, not proof artifacts",
179 + }
180 + width, height, data = load_image(path)
181 + expected_width = spec.get("width")
182 + expected_height = spec.get("height")
183 + results = []
184 + ok = True
185 + if expected_width is not None and int(expected_width) != width:
186 + ok = False
187 + results.append({"kind": "dimensions", "pass": False, "actual": [width, height], "expected": [expected_width, expected_height]})
188 + if expected_height is not None and int(expected_height) != height:
189 + ok = False
190 + if not results or results[-1].get("kind") != "dimensions":
191 + results.append({"kind": "dimensions", "pass": False, "actual": [width, height], "expected": [expected_width, expected_height]})
192 + for assertion in spec.get("assertions", []):
193 + item = evaluate_assertion(width, height, data, assertion)
194 + results.append(item)
195 + ok = ok and item["pass"]
196 + return {
197 + "ok": ok,
198 + "path": path,
199 + "source_kind": spec.get("source_kind", "readback"),
200 + "origin": spec.get("origin", "top-left"),
201 + "format": "RGBA8",
202 + "width": width,
203 + "height": height,
204 + "assertions": results,
205 + }
206 +
207 +
208 + def write_ppm(path, width, height, fill):
209 + rgb = bytes(fill[:3])
210 + with open(path, "wb") as f:
211 + f.write(("P6\n%d %d\n255\n" % (width, height)).encode("ascii"))
212 + f.write(rgb * width * height)
213 +
214 +
215 + def patch_rect_ppm(path, width, x, y, w, h, color):
216 + with open(path, "r+b") as f:
217 + data = f.read()
218 + header_end = data.find(b"\n255\n") + 5
219 + pixels = bytearray(data[header_end:])
220 + for yy in range(y, y + h):
221 + for xx in range(x, x + w):
222 + idx = (yy * width + xx) * 3
223 + pixels[idx:idx + 3] = bytes(color[:3])
224 + f.seek(header_end)
225 + f.write(pixels)
226 +
227 +
228 + def selftest():
229 + with tempfile.TemporaryDirectory() as tmp:
230 + on = os.path.join(tmp, "feature_on.ppm")
231 + off = os.path.join(tmp, "feature_off.ppm")
232 + write_ppm(on, 4, 4, PALETTE["background"])
233 + write_ppm(off, 4, 4, PALETTE["background"])
234 + patch_rect_ppm(on, 4, 1, 1, 2, 2, PALETTE["foreground"])
235 + spec = {
236 + "source_kind": "fixture",
237 + "width": 4,
238 + "height": 4,
239 + "origin": "top-left",
240 + "assertions": [
241 + {"kind": "point", "x": 1, "y": 1, "expected": "foreground"},
242 + {"kind": "rect", "x": 0, "y": 0, "w": 1, "h": 4, "expected": "background"},
243 + ],
244 + }
245 + passed = evaluate(on, spec)
246 + failed = evaluate(off, spec)
247 + screenshot = evaluate(on, {"source_kind": "screenshot", "assertions": []})
248 + assert passed["ok"], passed
249 + assert not failed["ok"], failed
250 + assert not screenshot["ok"], screenshot
251 + print("visual_assert selftest ok")
252 + return 0
253 +
254 +
255 + def main():
256 + parser = argparse.ArgumentParser()
257 + parser.add_argument("image", nargs="?")
258 + parser.add_argument("--spec", help="JSON assertion spec path")
259 + parser.add_argument("--json", dest="json_path")
260 + parser.add_argument("--selftest", action="store_true")
261 + args = parser.parse_args()
262 + if args.selftest:
263 + return selftest()
264 + if not args.image or not args.spec:
265 + parser.error("image and --spec are required unless --selftest is used")
266 + with open(args.spec, "r", encoding="utf-8") as f:
267 + spec = json.load(f)
268 + result = evaluate(args.image, spec)
269 + encoded = json.dumps(result, sort_keys=True)
270 + if args.json_path:
271 + with open(args.json_path, "w", encoding="utf-8") as f:
272 + f.write(encoded + "\n")
273 + else:
274 + print(encoded)
275 + return 0 if result.get("ok") else 1
276 +
277 +
278 + if __name__ == "__main__":
279 + sys.exit(main())