feat(review): 取证层多源化 + 视觉多图（整体+2x2 象限）
- 新增 evidence_model.py：统一证据模型（source_type + images 列表），访问器兼容旧 image_b64/image_mime 单图字段 - phases/evidence.py：新增 collect() 多源分发 seam（仅 CAD）、capture_images() 整体+等分象限多图（结束复位视口）、sub_windows() - phases/review.py：多图逐张下采样喂 LLM，payload 增加 source_type - config.py：EVIDENCE_DETAIL_TILES=4 / EVIDENCE_MAX_IMAGES=5（默认开启多图） - 新增 tests/test_evidence_model.py，更新 evidence/review 断言与 fakes 归一
This commit is contained in:
gjm committed
1 parent
22db93b6f5
commit
1c9fc72839
10 files changed
+396
-32
No files matched your search
@@ -11,6 +11,7 @@ from . import findings as findings_mod
|
||||
from .annotator import annotate, clear
|
||||
from .checklist import build_system_prompt, checklist_summary
|
||||
from .context import ReviewContext
|
||||
from .evidence_model import evidence_images
|
||||
from .mcp_client import McpError, McpToolError, result_json
|
||||
from .phases import evidence as evidence_phase
|
||||
from .phases import planning, report, review
|
||||
@@ -20,7 +21,8 @@ def _dump_evidence(evidences):
|
||||
out_dir = Path(config.REPORT_DIR)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / "evidence.json"
|
||||
trimmed = [{k: v for k, v in e.items() if k != "image_b64"} for e in evidences]
|
||||
drop = ("images", "image_b64", "image_mime")
|
||||
trimmed = [{k: v for k, v in e.items() if k not in drop} for e in evidences]
|
||||
path.write_text(json.dumps(trimmed, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
@@ -67,14 +69,14 @@ def run_review(ctx, args):
|
||||
f" - 图框 {idx}/{len(frames)} kind={frame.get('kind')} "
|
||||
f"sheet={frame.get('sheet_size')}"
|
||||
)
|
||||
ev = evidence_phase.collect_frame(ctx, idx, frame)
|
||||
ev = evidence_phase.collect(ctx, idx, frame)
|
||||
if ev is None:
|
||||
print(" [跳过] 缺少有效包围盒")
|
||||
continue
|
||||
evidences.append(ev)
|
||||
print(
|
||||
f" 图元 {(ev.get('entities') or {}).get('total')} 条，"
|
||||
f"截图 {'有' if ev.get('image_b64') else '无'}"
|
||||
f"截图 {len(evidence_images(ev))} 张"
|
||||
)
|
||||
|
||||
if args.plan_only:
|
||||
|
||||
@@ -121,6 +121,11 @@ IMAGE_FORMAT = os.environ.get("IMAGE_FORMAT", "png").strip().lower()
|
||||
# JPEG 质量（仅 IMAGE_FORMAT=jpeg 时生效）
|
||||
IMAGE_JPEG_QUALITY = int(os.environ.get("IMAGE_JPEG_QUALITY", "80"))
|
||||
|
||||
# 取证阶段每图框的局部截图数（0=仅整体；4=整体+2x2 象限）
|
||||
EVIDENCE_DETAIL_TILES = int(os.environ.get("EVIDENCE_DETAIL_TILES", "4"))
|
||||
# 每图框图片硬上限（整体 + 局部总张数）
|
||||
EVIDENCE_MAX_IMAGES = int(os.environ.get("EVIDENCE_MAX_IMAGES", "5"))
|
||||
|
||||
|
||||
def has_api_key() -> bool:
|
||||
return bool(API_KEY.strip())
|
||||
@@ -0,0 +1,88 @@
|
||||
"""统一证据模型：多来源（CAD / PDF / 图片）证据的字段常量与访问器。
|
||||
|
||||
设计要点：
|
||||
- evidence 是跨阶段 + 落盘的「线格式」数据，统一用 dict 承载（不用 dataclass）。
|
||||
- 图片统一为 `images: [{"b64","mime","label"}]`；访问器兼容旧的 `image_b64/image_mime`
|
||||
单图字段（历史 evidence.json 与旧测试构造仍可读）。
|
||||
- 本模块不含任何 CAD / 工具逻辑，可被 phases、findings、__main__、tests 共同引用。
|
||||
"""
|
||||
|
||||
SOURCE_CAD = "cad"
|
||||
SOURCE_PDF = "pdf"
|
||||
SOURCE_IMAGE = "image"
|
||||
|
||||
# 结构化证据键（CAD 侧各查询工具产出的信封）；下游 handle 收集 / 派生检查统一引用，
|
||||
# 避免在多处硬编码同一组键名。
|
||||
STRUCTURED_SOURCES = ("entities", "texts", "dimensions", "blocks", "polylines")
|
||||
|
||||
|
||||
def make_image(b64, mime="image/png", label=None):
|
||||
"""构造 images 列表元素。"""
|
||||
return {"b64": b64, "mime": mime or "image/png", "label": label}
|
||||
|
||||
|
||||
def source_type_of(container):
|
||||
"""读取来源类型，缺省视为 CAD。
|
||||
|
||||
container 可以是审查单元（unit）/ 图框 / evidence 等任何 dict。
|
||||
"""
|
||||
if isinstance(container, dict):
|
||||
value = container.get("source_type")
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return SOURCE_CAD
|
||||
|
||||
|
||||
def evidence_images(evidence):
|
||||
"""规范化证据里的图片列表（单点归一）。
|
||||
|
||||
优先读 `images`；否则回退旧的 `image_b64` / `image_mime` 单图字段。
|
||||
统一返回 `[{"b64","mime","label"}]`，过滤掉空 b64。
|
||||
"""
|
||||
if not isinstance(evidence, dict):
|
||||
return []
|
||||
images = []
|
||||
raw = evidence.get("images")
|
||||
if isinstance(raw, list):
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
b64 = item.get("b64")
|
||||
if not b64:
|
||||
continue
|
||||
images.append(
|
||||
{
|
||||
"b64": b64,
|
||||
"mime": item.get("mime") or "image/png",
|
||||
"label": item.get("label"),
|
||||
}
|
||||
)
|
||||
if images:
|
||||
return images
|
||||
# 回退：旧的单图字段
|
||||
b64 = evidence.get("image_b64")
|
||||
if b64:
|
||||
return [
|
||||
{
|
||||
"b64": b64,
|
||||
"mime": evidence.get("image_mime") or "image/png",
|
||||
"label": None,
|
||||
}
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def primary_image(evidence):
|
||||
"""返回首图（label == "overall" 优先），无图返回 None。"""
|
||||
images = evidence_images(evidence)
|
||||
if not images:
|
||||
return None
|
||||
for img in images:
|
||||
if img.get("label") == "overall":
|
||||
return img
|
||||
return images[0]
|
||||
|
||||
|
||||
def image_labels(evidence):
|
||||
"""返回图片标签列表（用于日志 / payload 说明）。"""
|
||||
return [img.get("label") for img in evidence_images(evidence)]
|
||||
@@ -1,14 +1,20 @@
|
||||
"""阶段 2：逐图框取证（确定性、串行）。
|
||||
"""阶段 2：逐审查单元取证（多源证据层 + 视觉多图）。
|
||||
|
||||
对每个图框：缩放到该框 → 取结构化数据（实体/文字/标注/块/多段线）→ 截取视口。
|
||||
对每个审查单元：取结构化数据（实体/文字/标注/块/多段线）+ 视觉截图（整体 + 可选局部）。
|
||||
结构化数据是权威依据，截图仅作视觉上下文。
|
||||
|
||||
多源设计：`collect()` 按审查单元来源分发；当前仅实现 CAD 源，PDF / 图片源预留 seam
|
||||
（未来在对应分支产出**同形** evidence，下游无需改动）。
|
||||
"""
|
||||
|
||||
import math
|
||||
import time
|
||||
|
||||
from .. import config
|
||||
from .. import config, evidence_model
|
||||
from ..mcp_client import result_images, result_json
|
||||
|
||||
_LABEL_OVERALL = "overall"
|
||||
|
||||
|
||||
def bbox_to_window(bbox):
|
||||
"""[minx,miny,maxx,maxy] -> window 对象。"""
|
||||
@@ -20,8 +26,77 @@ def bbox_to_window(bbox):
|
||||
}
|
||||
|
||||
|
||||
def sub_windows(window, tiles):
|
||||
"""把 window 等分为 tiles 个子窗口（tiles=4 → 2x2 象限）。
|
||||
|
||||
tiles<=1 返回 []。返回 [{"min_x",..,"max_x",..}, ...]，行优先（左上→右上→左下→右下）。
|
||||
"""
|
||||
if not tiles or tiles <= 1:
|
||||
return []
|
||||
cols = int(math.ceil(math.sqrt(tiles)))
|
||||
rows = int(math.ceil(tiles / cols))
|
||||
min_x, min_y = window["min_x"], window["min_y"]
|
||||
max_x, max_y = window["max_x"], window["max_y"]
|
||||
w = (max_x - min_x) / cols
|
||||
h = (max_y - min_y) / rows
|
||||
out = []
|
||||
for r in range(rows):
|
||||
for c in range(cols):
|
||||
if len(out) >= tiles:
|
||||
break
|
||||
out.append(
|
||||
{
|
||||
"min_x": min_x + c * w,
|
||||
"min_y": max_y - (r + 1) * h,
|
||||
"max_x": min_x + (c + 1) * w,
|
||||
"max_y": max_y - r * h,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _zoom_and_shot(ctx, window, margin=0.02):
|
||||
"""缩放到窗口、等待渲染、截一张图，返回 (b64, mime) 或 None。"""
|
||||
ctx.tools.call("zoom_to_window", dict(window, margin=margin))
|
||||
time.sleep(config.RENDER_WAIT)
|
||||
shot = ctx.tools.call("get_viewport_screenshot", {})
|
||||
images = result_images(shot)
|
||||
if not images:
|
||||
return None
|
||||
return images[0]
|
||||
|
||||
|
||||
def capture_images(ctx, window, *, tiles=0, max_images=None):
|
||||
"""采集视觉证据：整体一张 + 局部 tiles 张。
|
||||
|
||||
- 整体 label="overall"，局部 label="tile-1"..；行优先。
|
||||
- 结束时**复位到整体视口**，保证后续流程从整幅视图起手。
|
||||
- 受 max_images（默认 config.EVIDENCE_MAX_IMAGES）截断。
|
||||
"""
|
||||
if max_images is None:
|
||||
max_images = config.EVIDENCE_MAX_IMAGES
|
||||
images = []
|
||||
|
||||
def _add(shot, label):
|
||||
if shot and len(images) < max_images:
|
||||
images.append(evidence_model.make_image(shot[0], shot[1], label))
|
||||
|
||||
# 整体
|
||||
_add(_zoom_and_shot(ctx, window), _LABEL_OVERALL)
|
||||
|
||||
# 局部（等分象限）
|
||||
for i, sub in enumerate(sub_windows(window, tiles), 1):
|
||||
if len(images) >= max_images:
|
||||
break
|
||||
_add(_zoom_and_shot(ctx, sub), f"tile-{i}")
|
||||
|
||||
# 复位整体视口（不截图）
|
||||
ctx.tools.call("zoom_to_window", dict(window, margin=0.02))
|
||||
return images
|
||||
|
||||
|
||||
def collect_frame(ctx, index, frame):
|
||||
"""采集单个图框的证据；bbox 无效时返回 None。"""
|
||||
"""采集单个图框（CAD 源）的证据；bbox 无效时返回 None。"""
|
||||
bbox = frame.get("bbox")
|
||||
if not bbox or len(bbox) != 4:
|
||||
return None
|
||||
@@ -29,11 +104,7 @@ def collect_frame(ctx, index, frame):
|
||||
win = bbox_to_window(bbox)
|
||||
limit = ctx.frame_limit
|
||||
|
||||
# a. 缩放到图框（给 CAD 留渲染时间）
|
||||
ctx.tools.call("zoom_to_window", dict(win, margin=0.02))
|
||||
time.sleep(config.RENDER_WAIT)
|
||||
|
||||
# b. 结构化数据（权威）
|
||||
# a. 结构化数据（权威；按 window 查询，不依赖缩放）
|
||||
entities = result_json(ctx.tools.call("query_entities", {"window": win, "limit": limit}), {})
|
||||
texts = result_json(ctx.tools.call("get_text_content", {"window": win, "limit": limit}), {})
|
||||
dimensions = result_json(ctx.tools.call("get_dimensions", {"window": win, "limit": limit}), {})
|
||||
@@ -44,12 +115,11 @@ def collect_frame(ctx, index, frame):
|
||||
ctx.tools.call("get_polylines_info", {"window": win, "limit": limit}), {}
|
||||
)
|
||||
|
||||
# c. 视口截图（视觉上下文）
|
||||
shot = ctx.tools.call("get_viewport_screenshot", {})
|
||||
images = result_images(shot)
|
||||
image_b64, image_mime = images[0] if images else (None, None)
|
||||
# b. 视觉：整体 + 可选局部（多图）
|
||||
images = capture_images(ctx, win, tiles=config.EVIDENCE_DETAIL_TILES)
|
||||
|
||||
return {
|
||||
"source_type": evidence_model.source_type_of(frame),
|
||||
"index": index,
|
||||
"frame": frame,
|
||||
"window": win,
|
||||
@@ -58,6 +128,18 @@ def collect_frame(ctx, index, frame):
|
||||
"dimensions": dimensions,
|
||||
"blocks": blocks,
|
||||
"polylines": polylines,
|
||||
"image_b64": image_b64,
|
||||
"image_mime": image_mime,
|
||||
"images": images,
|
||||
}
|
||||
|
||||
|
||||
def collect(ctx, index, unit):
|
||||
"""按审查单元来源分发取证。
|
||||
|
||||
当前仅实现 CAD 源；PDF / 图片源预留（未来在对应分支接入离线解析 / 直接读图，
|
||||
产出与 CAD **同形**的 evidence：window=None、结构化键给空信封、images 带 label）。
|
||||
"""
|
||||
source = evidence_model.source_type_of(unit)
|
||||
if source == evidence_model.SOURCE_CAD:
|
||||
return collect_frame(ctx, index, unit)
|
||||
# TODO(pdf/image): 接入离线解析（DWG/PDF）与直接读图。
|
||||
return None
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
import json
|
||||
|
||||
from .. import evidence_model
|
||||
from ..cad_tools import READ_ONLY_TOOLS, is_write_tool
|
||||
from ..findings import parse_conclusion
|
||||
from ..llm_client import LlmClient, downsample_image_b64
|
||||
@@ -24,6 +25,7 @@ def evidence_payload(evidence):
|
||||
"""把证据整理成给 LLM 的 JSON（剔除 base64 图片，避免撑爆上下文）。"""
|
||||
frame = evidence.get("frame") or {}
|
||||
return {
|
||||
"source_type": evidence.get("source_type"),
|
||||
"frame_index": evidence.get("index"),
|
||||
"frame": {
|
||||
k: frame.get(k)
|
||||
@@ -72,9 +74,11 @@ def review_frame(ctx, evidence, system_prompt, log=print):
|
||||
+ json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
}
|
||||
]
|
||||
image_b64 = evidence.get("image_b64")
|
||||
if image_b64:
|
||||
nb, nm = downsample_image_b64(image_b64, evidence.get("image_mime") or "image/png")
|
||||
for img in evidence_model.evidence_images(evidence):
|
||||
nb, nm = downsample_image_b64(img["b64"], img.get("mime") or "image/png")
|
||||
label = img.get("label")
|
||||
if label:
|
||||
user_content.append({"type": "text", "text": f"（截图：{label}）"})
|
||||
user_content.append(LlmClient.image_content(nb, nm))
|
||||
|
||||
messages = [
|
||||
|
||||
Reference in new issue
Block a user