"""agent 侧虚拟工具 `inspect_region`:按**可读分辨率**放大指定区域。 动机(A 阶段收尾): 让 LLM 自己组合 `zoom_to_window` + `get_viewport_screenshot` 时—— ① 单张截图可能仍读不清(宽区域下采样后字太小); ② 「看清没」只能由 VLM 自评(易过度自信 / 幻觉)。 本模块把「放大」升级为**一个语义化工具**:LLM 只指认「要看哪里」,由 agent 复用 `tiling.py` 的标定数学算出该切几张、每张覆盖多大,**用构造保证可读**。 不变量: - 只调用**白名单内**的只读 MCP 工具(`zoom_to_window` / `get_viewport_screenshot`), 不写库、不改图元、不动图层; - 会改动视口 → 调用方(`phases/review.py`)负责在该图框审查结束后复位。 坐标约定: - 世界坐标:x 向右、y **向上**; - 截图内归一化坐标:cx 左→右、cy **上→下**(图像坐标,左上原点)—— `_point_box` 会翻转 y,这是最容易写错的一处。 """ from . import config, evidence_model, tiling from .llm_client import LlmClient, downsample_image_b64, image_size_b64 from .phases.evidence import zoom_and_shot TOOL_NAME = "inspect_region" # zoom 时四周留边比例(与 `evidence.zoom_and_shot` 的默认 margin 一致) _ZOOM_MARGIN = 0.02 # 送进瓦片规划前先把目标框外扩一点,避免把边界上的字切掉 _EDGE_PAD = 0.05 TOOL_SCHEMA = { "type": "function", "function": { "name": TOOL_NAME, "description": ( "放大并返回指定区域的**可读分辨率**截图。当你看不清已给截图里的文字/标注/细节时调用;" "会自动计算需要几张、每张覆盖多大,**无需你判断清晰度**。" "target 三选一:handle(实体句柄)/ window(世界坐标框)/ " "label+cx+cy(某张已给截图上的归一化位置)。" ), "parameters": { "type": "object", "properties": { "target": { "type": "object", "description": "要放大的位置(三种形态任选其一)", "properties": { "handle": { "type": "string", "description": '实体句柄,如 "3A4"', }, "window": { "type": "object", "description": "世界坐标框", "properties": { "min_x": {"type": "number"}, "min_y": {"type": "number"}, "max_x": {"type": "number"}, "max_y": {"type": "number"}, }, "required": ["min_x", "min_y", "max_x", "max_y"], }, "label": { "type": "string", "description": '已给截图的标签,如 "tile-3" 或之前返回的 "inspect-1"', }, "cx": { "type": "number", "description": "该截图内的归一化横坐标 0~1(左→右)", }, "cy": { "type": "number", "description": "该截图内的归一化纵坐标 0~1(上→下)", }, "span": { "type": "number", "description": "以该点为中心的视野跨度(占该截图宽/高的比例),默认 0.25", }, }, }, "purpose": { "type": "string", "enum": ["text", "line"], "description": "放大目的:text=读文字/标注(默认,更细);line=看走向/连接(更粗)", }, }, "required": ["target"], }, }, } class InspectState: """单图框内的「截图标签 → 世界可见范围」登记表。 初始登记取证阶段给的 `overall` / `tile-*`(它们的 `window` 字段), 之后每次 `inspect_region` 的新截图再追加 `inspect-1`、`inspect-2`…, 于是 LLM 可以用**上一轮返回的标签**继续下钻。 """ def __init__(self, evidence): self.labels = {} for img in evidence_model.evidence_images(evidence): label = img.get("label") box = tiling.norm_box(img.get("window")) if label and box is not None: self.labels[label] = box self.seq = 0 def register(self, box): """登记一张新截图,返回它的标签。""" self.seq += 1 label = f"inspect-{self.seq}" self.labels[label] = box return label def _text(s): return {"type": "text", "text": s} def readable_px(purpose): """按用途取 VLM 可读下限(像素)。""" return config.TEXT_READABLE_PX if purpose == "text" else config.LINE_READABLE_PX def _view_px(evidence): """绘图区**原始像素**(取 overall 图;无图/无 Pillow → None)。""" img = evidence_model.primary_image(evidence) if not img: return None return image_size_b64(img.get("b64")) def _bbox_for_handle(evidence, handle): """在已采集的结构化证据里按 handle 找 bbox;找不到返回 None。""" want = handle.strip().upper() for key in evidence_model.STRUCTURED_SOURCES: env = evidence.get(key) items = env.get("items") if isinstance(env, dict) else None for item in items or []: if not isinstance(item, dict): continue got = item.get("handle") if isinstance(got, str) and got.strip().upper() == want: box = tiling.norm_box(item.get("bbox")) if box is not None: return box return None def _point_box(base, cx, cy, span): """截图内归一化点 → 世界框。 `cx` 左→右;`cy` **上→下**(图像坐标),故世界 y 需翻转:`y = y2 - cy * H`。 """ x1, y1, x2, y2 = base w, h = x2 - x1, y2 - y1 cx = min(max(float(cx), 0.0), 1.0) cy = min(max(float(cy), 0.0), 1.0) span = min(max(float(span), 0.02), 1.0) px = x1 + cx * w py = y2 - cy * h hw, hh = w * span / 2.0, h * span / 2.0 return (px - hw, py - hh, px + hw, py + hh) def resolve_target(evidence, target, state): """解析 `target` → `(世界框, 人类可读说明)`;失败返回 `(None, 错误说明)`。""" if not isinstance(target, dict): return None, "target 必须是对象" handle = target.get("handle") if isinstance(handle, str) and handle.strip(): box = _bbox_for_handle(evidence, handle) if box is None: return None, ( f"未在已采集证据里找到 handle {handle} 的包围盒;" f"请改用 window 或 label 指认位置" ) return box, f"实体 {handle} 的包围盒" if target.get("window") is not None: box = tiling.window_to_box(target.get("window")) if box is None: return None, "window 非法:需要 min_x/min_y/max_x/max_y 四个数值" return box, "给定的世界坐标框" label = target.get("label") if isinstance(label, str) and label.strip(): label = label.strip() base = state.labels.get(label) if base is None: known = "、".join(sorted(state.labels)) or "(无)" return None, f"未知截图标签 {label};当前可用:{known}" try: cx = float(target.get("cx", 0.5)) cy = float(target.get("cy", 0.5)) span = float(target.get("span", 0.25)) except (TypeError, ValueError): return None, "cx / cy / span 必须是数值" return ( _point_box(base, cx, cy, span), f"截图 {label} 上 ({cx:.2f}, {cy:.2f}) 附近", ) return None, "target 需给出 handle / window / label 之一" def plan_tiles(box, scale, purpose, view_px, *, max_tiles, overlap, margin): """把目标框规划成若干 zoom 窗口。 返回 `(windows, note)`;`windows is None` 表示**硬失败**(区域过大,已中止)。 拿不到比例 / 绘图区像素时**退回单张**并如实告知(无法保证可读)。 """ s = tiling.world_per_px(scale, config.TEXT_HEIGHT_MM, readable_px(purpose)) eff = tiling.effective_px(view_px, config.IMAGE_MAX_EDGE) padded = tiling.expand_box(box, _EDGE_PAD) if padded is None or padded[2] <= padded[0] or padded[3] <= padded[1]: return None, "目标区域退化(宽或高为 0),无法放大" if s <= 0 or eff[0] <= 0 or eff[1] <= 0: win = tiling.box_to_window(padded) return [win], "(拿不到出图比例或绘图区像素,已退回单张;文字可能仍读不清)" cols, rows = tiling.tiles_for_box(padded, s * eff[0], s * eff[1], overlap) need = cols * rows if need > max_tiles: return None, ( f"该区域按可读分辨率需 {need} 张截图,超过单次上限 {max_tiles};" f"请缩小 target(例如用 label + cx + cy + span 指定更小范围)或分多次请求" ) wins = tiling.subdivide(padded, cols, rows, overlap, frame_win=None) if not wins: return None, "目标区域退化,无法放大" return wins, f"按 {purpose} 判据切为 {cols}×{rows} = {need} 张" def inspect_region(ctx, evidence, args, image_budget, state): """执行一次放大取证,返回 `(content, image_budget)`。""" args = args if isinstance(args, dict) else {} purpose = str(args.get("purpose") or "text").strip().lower() if purpose not in ("text", "line"): purpose = "text" box, note = resolve_target(evidence, args.get("target"), state) if box is None: return [_text(f"inspect_region 未执行:{note}")], image_budget view_px = _view_px(evidence) scale = tiling.frame_scale(evidence.get("frame")) wins, plan_note = plan_tiles( box, scale, purpose, view_px, max_tiles=config.INSPECT_MAX_TILES, overlap=config.TILE_OVERLAP, margin=_ZOOM_MARGIN, ) if wins is None: return [_text(f"inspect_region 未执行:{plan_note}")], image_budget content = [_text(f"inspect_region:{note};{plan_note}。")] taken = 0 for i, win in enumerate(wins, 1): if image_budget[0] <= 0: content.append(_text( f"(截图预算已用完,剩余 {len(wins) - i + 1} 张未拍;" f"请基于已有信息给出结论,不要再请求截图。)" )) break shot = zoom_and_shot(ctx, win, margin=_ZOOM_MARGIN) if not shot: continue image_budget[0] -= 1 visible = tiling.visible_box(tiling.window_to_box(win), view_px, _ZOOM_MARGIN) label = state.register(visible) content.append(_text( f"(放大截图 {label}:可见世界范围 " f"x[{visible[0]:.1f}, {visible[2]:.1f}] y[{visible[1]:.1f}, {visible[3]:.1f}])" )) b64, mime = downsample_image_b64(shot[0], shot[1]) content.append(LlmClient.image_content(b64, mime)) taken += 1 if taken == 0: content.append(_text("(未能取得截图。)")) return content, image_budget