mARTin-B78
ac8a6c7a24
The model only reports observations; expectations come from the sheet and the comparison is in code. Fail-safe: an unreachable checker or unreadable answer is a failure. Calibrated 14/14 on panels with known defects using the already loaded LLM; Ling 3.0 Flash VL (free) scored 8/14 with 6 unreadable answers. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
162 lines
8.5 KiB
Python
162 lines
8.5 KiB
Python
"""Automatic character image check: request shape, parsing, judging, and the route."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-imgcheck-test-config-"))
|
|
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
from core.character_image_check import build_messages, expected_from_sheet, judge, parse_observation
|
|
import routes.conversation as conversation
|
|
|
|
PNG = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
|
WOMAN = expected_from_sheet({"gender": "female"}, is_elf=False, age_category="adult")
|
|
GOOD_TURNAROUND = {
|
|
"figure_count": 3, "views_left_to_right": ["front", "side", "back"],
|
|
"all_feet_visible": True, "all_heads_visible": True, "same_person_in_all": True,
|
|
"apparent_gender": "female", "ears": "round", "wounds_or_blood": False, "background": "plain",
|
|
}
|
|
|
|
|
|
class RequestTests(unittest.TestCase):
|
|
def test_the_question_asks_for_observations_not_for_agreement(self):
|
|
text = build_messages("turnaround", [PNG])[0]["content"][0]["text"]
|
|
self.assertIn("views_left_to_right", text)
|
|
self.assertIn("Report only what is actually visible", text)
|
|
self.assertNotIn("female", text.split("keys:")[0]) # the expected gender is never revealed
|
|
|
|
def test_same_person_sends_both_images(self):
|
|
parts = build_messages("same_person", [PNG, PNG])[0]["content"]
|
|
self.assertEqual(sum(1 for p in parts if p["type"] == "image_url"), 2)
|
|
|
|
def test_wrong_image_counts_are_rejected(self):
|
|
with self.assertRaises(ValueError):
|
|
build_messages("same_person", [PNG])
|
|
with self.assertRaises(ValueError):
|
|
build_messages("portrait", [PNG, PNG])
|
|
|
|
|
|
class ParseTests(unittest.TestCase):
|
|
def test_reads_fenced_json_and_drops_unknown_keys(self):
|
|
raw = '<think>hmm</think>```json\n{"apparent_gender":"male","extra":1}\n```'
|
|
self.assertEqual(parse_observation(raw, "portrait"), {"apparent_gender": "male"})
|
|
|
|
def test_garbage_gives_an_empty_observation(self):
|
|
self.assertEqual(parse_observation("I cannot tell.", "portrait"), {})
|
|
|
|
|
|
class JudgeTests(unittest.TestCase):
|
|
def test_a_correct_turnaround_passes(self):
|
|
self.assertTrue(judge("turnaround", GOOD_TURNAROUND, WOMAN)["passed"])
|
|
|
|
def test_three_front_views_fail(self):
|
|
out = judge("turnaround", dict(GOOD_TURNAROUND, views_left_to_right=["front", "front", "front"]), WOMAN)
|
|
self.assertFalse(out["passed"])
|
|
self.assertTrue(any("views" in f for f in out["failures"]))
|
|
|
|
def test_cut_off_feet_fail(self):
|
|
out = judge("turnaround", dict(GOOD_TURNAROUND, all_feet_visible=False), WOMAN)
|
|
self.assertIn("feet cut off", out["failures"])
|
|
|
|
def test_a_man_where_the_sheet_says_woman_fails(self):
|
|
out = judge("scene", {"main_person_gender": "male", "ears": "round", "wounds_or_blood": False}, WOMAN)
|
|
self.assertEqual(out["failures"], ["gender: shows male, sheet says female"])
|
|
|
|
def test_elf_ears_on_a_human_fail_but_an_elf_needs_them(self):
|
|
self.assertFalse(judge("portrait", {"person_count": 1, "apparent_gender": "female", "ears": "pointed",
|
|
"wounds_or_blood": False, "background": "plain"}, WOMAN)["passed"])
|
|
elf = expected_from_sheet({"gender": "female"}, is_elf=True, age_category="adult")
|
|
self.assertTrue(judge("portrait", {"person_count": 1, "apparent_gender": "female", "ears": "pointed",
|
|
"wounds_or_blood": False, "background": "plain"}, elf)["passed"])
|
|
|
|
def test_hidden_ears_are_not_held_against_the_image(self):
|
|
self.assertTrue(judge("portrait", {"person_count": 1, "apparent_gender": "female", "ears": "hidden",
|
|
"wounds_or_blood": False, "background": "plain"}, WOMAN)["passed"])
|
|
|
|
def test_scenery_behind_a_studio_portrait_fails(self):
|
|
out = judge("portrait", {"person_count": 1, "apparent_gender": "female", "ears": "round",
|
|
"wounds_or_blood": False, "background": "scenery"}, WOMAN)
|
|
self.assertFalse(out["passed"])
|
|
|
|
def test_wounds_fail(self):
|
|
out = judge("turnaround", dict(GOOD_TURNAROUND, wounds_or_blood=True), WOMAN)
|
|
self.assertIn("wounds or blood visible", out["failures"])
|
|
|
|
def test_an_observation_the_model_did_not_give_is_a_failure(self):
|
|
self.assertFalse(judge("turnaround", {}, WOMAN)["passed"])
|
|
|
|
def test_no_gender_on_the_sheet_means_gender_is_not_judged(self):
|
|
unknown = expected_from_sheet({}, is_elf=False, age_category="adult")
|
|
out = judge("scene", {"main_person_gender": "male", "ears": "round", "wounds_or_blood": False}, unknown)
|
|
self.assertTrue(out["passed"])
|
|
|
|
def test_a_different_person_from_the_profile_fails(self):
|
|
self.assertFalse(judge("same_person", {"same_person": False}, WOMAN)["passed"])
|
|
|
|
|
|
class RouteTests(unittest.TestCase):
|
|
def setUp(self):
|
|
app = FastAPI()
|
|
app.include_router(conversation.router)
|
|
self.client = TestClient(app)
|
|
|
|
def answer(self, content, status=200):
|
|
r = mock.Mock(); r.status_code = status
|
|
r.json.return_value = {"choices": [{"message": {"content": content}}]}
|
|
r.text = content
|
|
return r
|
|
|
|
def test_judges_the_models_observation_against_the_sheet(self):
|
|
with mock.patch.object(conversation, "_post_llm_chat_completion",
|
|
return_value=self.answer('{"main_person_gender":"male","ears":"round","wounds_or_blood":false}')):
|
|
out = self.client.post("/api/character-image-check",
|
|
json={"kind": "scene", "images": [PNG], "sheet": {"gender": "female"}}).json()
|
|
self.assertFalse(out["passed"])
|
|
self.assertEqual(out["failures"], ["gender: shows male, sheet says female"])
|
|
self.assertEqual(out["expected"]["gender"], "female")
|
|
|
|
def test_a_hedged_species_is_judged_as_human(self):
|
|
sheet = {"gender": "female", "race_species": "Mensch (oder Halbelfe, je nach Kontext, hier nicht spezifiziert)"}
|
|
with mock.patch.object(conversation, "_post_llm_chat_completion",
|
|
return_value=self.answer('{"main_person_gender":"female","ears":"pointed","wounds_or_blood":false}')):
|
|
out = self.client.post("/api/character-image-check",
|
|
json={"kind": "scene", "images": [PNG], "sheet": sheet}).json()
|
|
self.assertEqual(out["expected"]["ears"], "round")
|
|
self.assertIn("ears: pointed, but the character is not an elf", out["failures"])
|
|
|
|
def test_thinking_is_switched_off_and_dropped_if_the_backend_refuses_it(self):
|
|
good = self.answer('{"main_person_gender":"female","ears":"round","wounds_or_blood":false}')
|
|
with mock.patch.object(conversation, "_post_llm_chat_completion",
|
|
side_effect=[self.answer("unsupported", status=400), good]) as llm:
|
|
out = self.client.post("/api/character-image-check",
|
|
json={"kind": "scene", "images": [PNG], "sheet": {"gender": "female"}}).json()
|
|
self.assertTrue(out["passed"])
|
|
self.assertEqual(llm.call_args_list[0].args[1]["chat_template_kwargs"], {"enable_thinking": False})
|
|
self.assertNotIn("chat_template_kwargs", llm.call_args_list[1].args[1])
|
|
|
|
def test_an_unreachable_checker_is_a_failure_not_a_pass(self):
|
|
with mock.patch.object(conversation, "_post_llm_chat_completion", side_effect=RuntimeError("down")):
|
|
out = self.client.post("/api/character-image-check",
|
|
json={"kind": "portrait", "images": [PNG], "sheet": {}}).json()
|
|
self.assertFalse(out["passed"])
|
|
self.assertTrue(out["failures"][0].startswith("checker unavailable"))
|
|
|
|
def test_bad_requests_are_rejected_before_any_model_call(self):
|
|
with mock.patch.object(conversation, "_post_llm_chat_completion") as llm:
|
|
for body in ({"kind": "nope", "images": [PNG]},
|
|
{"kind": "portrait", "images": ["http://evil/x.png"]},
|
|
{"kind": "portrait", "images": []},
|
|
{"kind": "same_person", "images": [PNG]}):
|
|
with self.subTest(body=str(body)[:50]):
|
|
self.assertEqual(self.client.post("/api/character-image-check", json=body).status_code, 400)
|
|
llm.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|