feat(review): 审查结论改为结构化 JSON,移除二次 LLM 抽取

- checklist.py:系统提示加入结构化输出契约(只输出 {summary, issues[]},
  引用 handle 必须真实存在、不得编造)
- review.py:review_frame 返回 {summary, issues, raw};仅在「不再请求工具」的回合
  解析结论,中间工具回合的 content 永不解析;强制收口那次 chat 不传 tools
- report.py:删除 extract_issues 与 _EXTRACT_SYSTEM(不再需要第二次 LLM 调用);
  新增 format_finding(用 .get,顺带修复缺 frame 键 KeyError);save_report 的
  issues 省略时从 findings 拍平
- __main__.py:主流程改为 确定性检查 → review_frame → build_finding(合并+校验),
  issues 由 flatten_issues 派生

顺带修复既有 gap:summarize 硬下标 KeyError、extract_issues 单行围栏解析失败。
This commit is contained in:
gjm committed 2026-10-14 19:53:41 +08:00
1 parent f6d61f8d07
commit f7fa8f14d9
8 files changed
+224 -140

No files matched your search

+1 -1
View File
@@ -101,7 +101,7 @@ class FakeLlmClient:
- `script` :`chat()` 依次弹出的响应队列;弹空后回落到 `text`。
- `text` :`chat()` / `chat_text()` 的兜底文本。
- `text_script` :`chat_text()` 专用队列(便于同时喂 summarize 与 extract_issues)。
- `text_script` :`chat_text()` 专用队列(便于连续喂多次 `chat_text` 调用)。
"""
def __init__(self, script=None, text="", text_script=None, model="fake-model"):
+16
View File
@@ -65,6 +65,22 @@ def test_build_system_prompt_tool_hint_toggle():
assert "只读" not in without
def test_build_system_prompt_requires_structured_json_output():
# allow_tools=False 以隔离「工具提示块」里的「只读」字样
prompt = build_system_prompt(DEFAULT_CHECKLIST, allow_tools=False)
assert "JSON" in prompt
assert '"summary"' in prompt
assert '"issues"' in prompt
assert '"severity"' in prompt and '"category"' in prompt and '"handle"' in prompt
# 结构化契约本身不得含「只读」,否则会破坏 allow_tools 开关的判别
assert "只读" not in prompt
def test_build_system_prompt_forbids_fabricated_handle():
prompt = build_system_prompt(DEFAULT_CHECKLIST)
assert "不要编造" in prompt
def test_checklist_summary_mentions_name_and_count():
summary = checklist_summary(DEFAULT_CHECKLIST)
assert DEFAULT_CHECKLIST["name"] in summary
+72 -70
View File
@@ -2,19 +2,64 @@
import json
import pytest
from dwg_review.phases.report import extract_issues, save_report, summarize
from dwg_review.phases.report import format_finding, save_report, summarize
from tests.fakes import FakeLlmClient, make_ctx
def issue(**over):
base = {
"frame": 1, "category": "文字", "severity": "high", "message": "文字重叠",
"handle": "A1", "source": "llm", "verified": True,
}
base.update(over)
return base
def findings_fixture():
return [
{"frame": 1, "sheet": "A1", "finding": "图框1:文字重叠"},
{"frame": 2, "sheet": "A3", "finding": "图框2:未发现明显问题"},
{
"frame": 1, "sheet": "A1", "summary": "该图框存在文字重叠",
"issues": [issue()],
"counts": {"deterministic": 0, "llm": 1, "unverified": 0},
},
{
"frame": 2, "sheet": "A3", "summary": "未发现明显问题",
"issues": [],
"counts": {"deterministic": 0, "llm": 0, "unverified": 0},
},
]
# ---------------------------------------------------------------------------
# format_finding
# ---------------------------------------------------------------------------
def test_format_finding_renders_summary_and_issues():
text = format_finding(findings_fixture()[0])
assert "图框 1" in text and "图幅 A1" in text
assert "该图框存在文字重叠" in text
assert "[high][LLM] 文字重叠" in text
assert "handle=A1" in text
def test_format_finding_marks_deterministic_source():
finding = {"frame": 1, "sheet": "A1", "summary": "s",
"issues": [issue(source="deterministic", handle="Z1")]}
assert "[确定性]" in format_finding(finding)
def test_format_finding_marks_unverified():
finding = {"frame": 1, "sheet": "A1", "summary": "s",
"issues": [issue(handle=None, verified=False)]}
text = format_finding(finding)
assert "未在证据中定位" in text
assert "handle=" not in text
def test_format_finding_tolerates_missing_keys():
text = format_finding({})
assert "图框 None" in text # 不再 KeyError(修复既有硬下标 gap)
# ---------------------------------------------------------------------------
# summarize
# ---------------------------------------------------------------------------
@@ -31,7 +76,16 @@ def test_summarize_uses_chat_text():
ctx = make_ctx(llm=llm)
assert summarize(ctx, findings_fixture(), log=lambda *a: None) == "## 总报告"
assert len(llm.calls) == 1
assert "图框1" in llm.calls[0]["user"]
assert "图框 1" in llm.calls[0]["user"]
assert "图框 2" in llm.calls[0]["user"]
def test_summarize_input_includes_issue_lines():
llm = FakeLlmClient(text="x")
ctx = make_ctx(llm=llm)
summarize(ctx, findings_fixture(), log=lambda *a: None)
user = llm.calls[0]["user"]
assert "[high][LLM] 文字重叠" in user
def test_summarize_falls_back_to_raw_input_on_error():
@@ -42,70 +96,16 @@ def test_summarize_falls_back_to_raw_input_on_error():
logs = []
ctx = make_ctx(llm=Boom())
out = summarize(ctx, findings_fixture(), log=logs.append)
assert "图框1" in out and "图框2" in out
assert "图框 1" in out and "图框 2" in out
assert any("汇总失败" in line for line in logs)
def test_summarize_missing_frame_key_known_gap():
"""已知瑕疵:summarize 用硬下标 f['frame'],findings 缺该键直接 KeyError。"""
ctx = make_ctx(llm=FakeLlmClient(text="x"))
with pytest.raises(KeyError):
summarize(ctx, [{"finding": "缺 frame 键"}], log=lambda *a: None)
# ---------------------------------------------------------------------------
# extract_issues
# ---------------------------------------------------------------------------
def test_extract_issues_empty_returns_empty_list():
llm = FakeLlmClient(text_script=["[]"])
def test_summarize_tolerates_finding_without_frame_key():
# 修复既有硬下标 gap:缺 frame 键不再 KeyError
llm = FakeLlmClient(text="ok")
ctx = make_ctx(llm=llm)
assert extract_issues(ctx, [], log=lambda *a: None) == []
assert llm.calls == []
def test_extract_issues_parses_json_array():
payload = '[{"frame": 1, "handle": "A", "severity": "high", "message": "x"}]'
llm = FakeLlmClient(text_script=[payload])
ctx = make_ctx(llm=llm)
issues = extract_issues(ctx, findings_fixture(), log=lambda *a: None)
assert issues == [{"frame": 1, "handle": "A", "severity": "high", "message": "x"}]
def test_extract_issues_strips_code_fence():
fenced = '```json\n[{"frame": 1, "message": "y"}]\n```'
llm = FakeLlmClient(text_script=[fenced])
ctx = make_ctx(llm=llm)
assert extract_issues(ctx, findings_fixture(), log=lambda *a: None) == [
{"frame": 1, "message": "y"}
]
def test_extract_issues_returns_empty_on_invalid_json():
logs = []
llm = FakeLlmClient(text_script=["这不是 JSON"])
ctx = make_ctx(llm=llm)
assert extract_issues(ctx, findings_fixture(), log=logs.append) == []
assert any("结构化抽取失败" in line for line in logs)
def test_extract_issues_returns_empty_when_not_a_list():
llm = FakeLlmClient(text_script=['{"frame": 1}'])
ctx = make_ctx(llm=llm)
assert extract_issues(ctx, findings_fixture(), log=lambda *a: None) == []
def test_extract_issues_single_line_fence_known_gap():
"""已知瑕疵:围栏剥离依赖换行,单行 ```json[...]``` 会残留 json 前缀 → 解析失败。"""
llm = FakeLlmClient(text_script=['```json[{"frame": 1, "message": "z"}]```'])
ctx = make_ctx(llm=llm)
assert extract_issues(ctx, findings_fixture(), log=lambda *a: None) == []
def test_extract_issues_missing_frame_key_known_gap():
"""已知瑕疵:extract_issues 同样用硬下标 f['frame'],findings 缺该键直接 KeyError。"""
ctx = make_ctx(llm=FakeLlmClient(text="[]"))
with pytest.raises(KeyError):
extract_issues(ctx, [{"finding": "缺 frame 键"}], log=lambda *a: None)
out = summarize(ctx, [{"summary": "无 frame 键", "issues": []}], log=lambda *a: None)
assert out == "ok"
# ---------------------------------------------------------------------------
@@ -114,7 +114,7 @@ def test_extract_issues_missing_frame_key_known_gap():
def test_save_report_writes_md_and_json(tmp_reports):
findings = findings_fixture()
plan = {"frames": [{"bbox": [0, 0, 10, 10]}]}
issues = [{"frame": 1, "handle": "A", "message": "x"}]
issues = [issue()]
md_path, json_path = save_report("# 报告", findings, plan, issues, log=lambda *a: None)
@@ -127,9 +127,11 @@ def test_save_report_writes_md_and_json(tmp_reports):
assert "generated_at" in data
def test_save_report_issues_default_to_empty(tmp_reports):
_, json_path = save_report("md", findings_fixture(), {"frames": []}, log=lambda *a: None)
assert json.loads(json_path.read_text(encoding="utf-8"))["issues"] == []
def test_save_report_derives_issues_from_findings(tmp_reports):
findings = findings_fixture()
_, json_path = save_report("md", findings, {"frames": []}, log=lambda *a: None)
data = json.loads(json_path.read_text(encoding="utf-8"))
assert data["issues"] == [issue()] # 拍平自 findings
def test_save_report_creates_missing_directory(tmp_path, monkeypatch):
+41 -5
View File
@@ -111,11 +111,12 @@ def test_readonly_empty_response_marker():
# ---------------------------------------------------------------------------
# review_frame
# ---------------------------------------------------------------------------
def test_review_frame_returns_text_when_no_tools():
def test_review_frame_returns_summary_when_no_tools():
llm = FakeLlmClient(script=[llm_text("结论A")])
ctx = make_ctx(client=FakeMcpClient(), llm=llm)
out = review_frame(ctx, make_evidence(), "系统提示", log=lambda *a: None)
assert out == "结论A"
assert out["summary"] == "结论A" # 非 JSON → 降级为纯文本 summary
assert out["issues"] == []
assert len(llm.calls) == 1
assert llm.calls[0]["tools"] is not None # 传了只读工具 schema
@@ -147,7 +148,7 @@ def test_review_frame_runs_tool_then_concludes():
client = FakeMcpClient(responses={"query_entities": ok_text({"total": 1})})
ctx = make_ctx(client=client, llm=llm)
out = review_frame(ctx, make_evidence(), "sys", log=lambda *a: None)
assert out == "结论B"
assert out["summary"] == "结论B"
assert len(llm.calls) == 2
assert client.called_names() == ["query_entities"]
@@ -179,7 +180,7 @@ def test_review_frame_force_closes_at_iteration_limit():
)
ctx = make_ctx(client=FakeMcpClient(), llm=llm, max_iters=1)
out = review_frame(ctx, make_evidence(), "sys", log=lambda *a: None)
assert out == "收口结论"
assert out["summary"] == "收口结论"
last_msg = llm.calls[-1]["messages"][-1]
assert last_msg["role"] == "user"
assert "上限" in last_msg["content"]
@@ -189,7 +190,7 @@ def test_review_frame_with_zero_iters_forces_immediate_close():
llm = FakeLlmClient(script=[llm_text("直接结论")])
ctx = make_ctx(client=FakeMcpClient(), llm=llm, max_iters=0)
out = review_frame(ctx, make_evidence(), "sys", log=lambda *a: None)
assert out == "直接结论"
assert out["summary"] == "直接结论"
assert len(llm.calls) == 1
@@ -207,3 +208,38 @@ def test_review_frame_payload_is_json_serializable():
review_frame(ctx, make_evidence(), "sys", log=lambda *a: None)
text = llm.calls[0]["messages"][1]["content"][0]["text"]
json.loads(text.split(":\n", 1)[1]) # 不抛异常即通过
def test_review_frame_parses_structured_json_conclusion():
text = (
'{"summary": "整体尚可", "issues": ['
'{"category": "几何", "severity": "medium", "message": "零长度", "handle": "A1"}]}'
)
llm = FakeLlmClient(script=[llm_text(text)])
ctx = make_ctx(client=FakeMcpClient(), llm=llm)
out = review_frame(ctx, make_evidence(), "sys", log=lambda *a: None)
assert out["summary"] == "整体尚可"
assert out["issues"] == [
{"category": "几何", "severity": "medium", "message": "零长度", "handle": "A1"}
]
def test_review_frame_parses_fenced_json():
llm = FakeLlmClient(script=[llm_text('```json\n{"summary": "s", "issues": []}\n```')])
ctx = make_ctx(client=FakeMcpClient(), llm=llm)
out = review_frame(ctx, make_evidence(), "sys", log=lambda *a: None)
assert out["summary"] == "s"
assert out["issues"] == []
def test_review_frame_intermediate_tool_turn_json_not_parsed():
# 中间工具回合的 content 里即使塞了 JSON,也不能被当成最终结论
intermediate = '{"summary": "这不是最终结论", "issues": [{"message": "x"}]}'
llm = FakeLlmClient(script=[
llm_calls([("query_entities", {})], content=intermediate),
llm_text('{"summary": "最终结论", "issues": []}'),
])
ctx = make_ctx(client=FakeMcpClient(), llm=llm)
out = review_frame(ctx, make_evidence(), "sys", log=lambda *a: None)
assert out["summary"] == "最终结论"
assert out["issues"] == []