288 lines
12 KiB
Python
288 lines
12 KiB
Python
|
"""agent 侧虚拟工具 `inspect_region`：按**可读分辨率**放大指定区域。
|
|||
|
|
|
|||
|
|
动机（A 阶段收尾）：
|
|||
|
|
让 LLM 自己组合 `zoom_to_window` + `get_viewport_screenshot` 时——
|
|||
|
|
① 单张截图可能仍读不清（宽区域下采样后字太小）；
|
|||
|
|
② 「看清没」只能由 VLM 自评（易过度自信 / 幻觉）。
|
|||
|
|
本模块把「放大」升级为**一个语义化工具**：LLM 只指认「要看哪里」，由 agent 复用
|
|||
|
|
`tiling.py` 的标定数学算出该切几张、每张覆盖多大，**用构造保证可读**。
|
|||
|
|
|
|||
|
|
不变量：
|
|||
|
|
- 只调用**白名单内**的只读 MCP 工具（`zoom_to_window` / `get_viewport_screenshot`），
|
|||
|
|
不写库、不改图元、不动图层；
|
|||
|
|
- 会改动视口 → 调用方（`phases/review.py`）负责在该图框审查结束后复位。
|
|||
|
|
|
|||
|
|
坐标约定：
|
|||
|
|
- 世界坐标：x 向右、y **向上**；
|
|||
|
|
- 截图内归一化坐标：cx 左→右、cy **上→下**（图像坐标，左上原点）——
|
|||
|
|
`_point_box` 会翻转 y，这是最容易写错的一处。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from . import config, evidence_model, tiling
|
|||
|
|
from .llm_client import LlmClient, downsample_image_b64, image_size_b64
|
|||
|
|
from .phases.evidence import zoom_and_shot
|
|||
|
|
|
|||
|
|
TOOL_NAME = "inspect_region"
|
|||
|
|
|
|||
|
|
# zoom 时四周留边比例（与 `evidence.zoom_and_shot` 的默认 margin 一致）
|
|||
|
|
_ZOOM_MARGIN = 0.02
|
|||
|
|
|
|||
|
|
# 送进瓦片规划前先把目标框外扩一点，避免把边界上的字切掉
|
|||
|
|
_EDGE_PAD = 0.05
|
|||
|
|
|
|||
|
|
TOOL_SCHEMA = {
|
|||
|
|
"type": "function",
|
|||
|
|
"function": {
|
|||
|
|
"name": TOOL_NAME,
|
|||
|
|
"description": (
|
|||
|
|
"放大并返回指定区域的**可读分辨率**截图。当你看不清已给截图里的文字/标注/细节时调用；"
|
|||
|
|
"会自动计算需要几张、每张覆盖多大，**无需你判断清晰度**。"
|
|||
|
|
"target 三选一：handle（实体句柄）/ window（世界坐标框）/ "
|
|||
|
|
"label+cx+cy（某张已给截图上的归一化位置）。"
|
|||
|
|
),
|
|||
|
|
"parameters": {
|
|||
|
|
"type": "object",
|
|||
|
|
"properties": {
|
|||
|
|
"target": {
|
|||
|
|
"type": "object",
|
|||
|
|
"description": "要放大的位置（三种形态任选其一）",
|
|||
|
|
"properties": {
|
|||
|
|
"handle": {
|
|||
|
|
"type": "string",
|
|||
|
|
"description": '实体句柄，如 "3A4"',
|
|||
|
|
},
|
|||
|
|
"window": {
|
|||
|
|
"type": "object",
|
|||
|
|
"description": "世界坐标框",
|
|||
|
|
"properties": {
|
|||
|
|
"min_x": {"type": "number"},
|
|||
|
|
"min_y": {"type": "number"},
|
|||
|
|
"max_x": {"type": "number"},
|
|||
|
|
"max_y": {"type": "number"},
|
|||
|
|
},
|
|||
|
|
"required": ["min_x", "min_y", "max_x", "max_y"],
|
|||
|
|
},
|
|||
|
|
"label": {
|
|||
|
|
"type": "string",
|
|||
|
|
"description": '已给截图的标签，如 "tile-3" 或之前返回的 "inspect-1"',
|
|||
|
|
},
|
|||
|
|
"cx": {
|
|||
|
|
"type": "number",
|
|||
|
|
"description": "该截图内的归一化横坐标 0~1（左→右）",
|
|||
|
|
},
|
|||
|
|
"cy": {
|
|||
|
|
"type": "number",
|
|||
|
|
"description": "该截图内的归一化纵坐标 0~1（上→下）",
|
|||
|
|
},
|
|||
|
|
"span": {
|
|||
|
|
"type": "number",
|
|||
|
|
"description": "以该点为中心的视野跨度（占该截图宽/高的比例），默认 0.25",
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
"purpose": {
|
|||
|
|
"type": "string",
|
|||
|
|
"enum": ["text", "line"],
|
|||
|
|
"description": "放大目的：text=读文字/标注（默认，更细）；line=看走向/连接（更粗）",
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
"required": ["target"],
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
class InspectState:
|
|||
|
|
"""单图框内的「截图标签 → 世界可见范围」登记表。
|
|||
|
|
|
|||
|
|
初始登记取证阶段给的 `overall` / `tile-*`（它们的 `window` 字段），
|
|||
|
|
之后每次 `inspect_region` 的新截图再追加 `inspect-1`、`inspect-2`…，
|
|||
|
|
于是 LLM 可以用**上一轮返回的标签**继续下钻。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, evidence):
|
|||
|
|
self.labels = {}
|
|||
|
|
for img in evidence_model.evidence_images(evidence):
|
|||
|
|
label = img.get("label")
|
|||
|
|
box = tiling.norm_box(img.get("window"))
|
|||
|
|
if label and box is not None:
|
|||
|
|
self.labels[label] = box
|
|||
|
|
self.seq = 0
|
|||
|
|
|
|||
|
|
def register(self, box):
|
|||
|
|
"""登记一张新截图，返回它的标签。"""
|
|||
|
|
self.seq += 1
|
|||
|
|
label = f"inspect-{self.seq}"
|
|||
|
|
self.labels[label] = box
|
|||
|
|
return label
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _text(s):
|
|||
|
|
return {"type": "text", "text": s}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def readable_px(purpose):
|
|||
|
|
"""按用途取 VLM 可读下限（像素）。"""
|
|||
|
|
return config.TEXT_READABLE_PX if purpose == "text" else config.LINE_READABLE_PX
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _view_px(evidence):
|
|||
|
|
"""绘图区**原始像素**（取 overall 图；无图/无 Pillow → None）。"""
|
|||
|
|
img = evidence_model.primary_image(evidence)
|
|||
|
|
if not img:
|
|||
|
|
return None
|
|||
|
|
return image_size_b64(img.get("b64"))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _bbox_for_handle(evidence, handle):
|
|||
|
|
"""在已采集的结构化证据里按 handle 找 bbox；找不到返回 None。"""
|
|||
|
|
want = handle.strip().upper()
|
|||
|
|
for key in evidence_model.STRUCTURED_SOURCES:
|
|||
|
|
env = evidence.get(key)
|
|||
|
|
items = env.get("items") if isinstance(env, dict) else None
|
|||
|
|
for item in items or []:
|
|||
|
|
if not isinstance(item, dict):
|
|||
|
|
continue
|
|||
|
|
got = item.get("handle")
|
|||
|
|
if isinstance(got, str) and got.strip().upper() == want:
|
|||
|
|
box = tiling.norm_box(item.get("bbox"))
|
|||
|
|
if box is not None:
|
|||
|
|
return box
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _point_box(base, cx, cy, span):
|
|||
|
|
"""截图内归一化点 → 世界框。
|
|||
|
|
|
|||
|
|
`cx` 左→右；`cy` **上→下**（图像坐标），故世界 y 需翻转：`y = y2 - cy * H`。
|
|||
|
|
"""
|
|||
|
|
x1, y1, x2, y2 = base
|
|||
|
|
w, h = x2 - x1, y2 - y1
|
|||
|
|
cx = min(max(float(cx), 0.0), 1.0)
|
|||
|
|
cy = min(max(float(cy), 0.0), 1.0)
|
|||
|
|
span = min(max(float(span), 0.02), 1.0)
|
|||
|
|
px = x1 + cx * w
|
|||
|
|
py = y2 - cy * h
|
|||
|
|
hw, hh = w * span / 2.0, h * span / 2.0
|
|||
|
|
return (px - hw, py - hh, px + hw, py + hh)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def resolve_target(evidence, target, state):
|
|||
|
|
"""解析 `target` → `(世界框, 人类可读说明)`；失败返回 `(None, 错误说明)`。"""
|
|||
|
|
if not isinstance(target, dict):
|
|||
|
|
return None, "target 必须是对象"
|
|||
|
|
|
|||
|
|
handle = target.get("handle")
|
|||
|
|
if isinstance(handle, str) and handle.strip():
|
|||
|
|
box = _bbox_for_handle(evidence, handle)
|
|||
|
|
if box is None:
|
|||
|
|
return None, (
|
|||
|
|
f"未在已采集证据里找到 handle {handle} 的包围盒；"
|
|||
|
|
f"请改用 window 或 label 指认位置"
|
|||
|
|
)
|
|||
|
|
return box, f"实体 {handle} 的包围盒"
|
|||
|
|
|
|||
|
|
if target.get("window") is not None:
|
|||
|
|
box = tiling.window_to_box(target.get("window"))
|
|||
|
|
if box is None:
|
|||
|
|
return None, "window 非法：需要 min_x/min_y/max_x/max_y 四个数值"
|
|||
|
|
return box, "给定的世界坐标框"
|
|||
|
|
|
|||
|
|
label = target.get("label")
|
|||
|
|
if isinstance(label, str) and label.strip():
|
|||
|
|
label = label.strip()
|
|||
|
|
base = state.labels.get(label)
|
|||
|
|
if base is None:
|
|||
|
|
known = "、".join(sorted(state.labels)) or "（无）"
|
|||
|
|
return None, f"未知截图标签 {label}；当前可用：{known}"
|
|||
|
|
try:
|
|||
|
|
cx = float(target.get("cx", 0.5))
|
|||
|
|
cy = float(target.get("cy", 0.5))
|
|||
|
|
span = float(target.get("span", 0.25))
|
|||
|
|
except (TypeError, ValueError):
|
|||
|
|
return None, "cx / cy / span 必须是数值"
|
|||
|
|
return (
|
|||
|
|
_point_box(base, cx, cy, span),
|
|||
|
|
f"截图 {label} 上 ({cx:.2f}, {cy:.2f}) 附近",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return None, "target 需给出 handle / window / label 之一"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def plan_tiles(box, scale, purpose, view_px, *, max_tiles, overlap, margin):
|
|||
|
|
"""把目标框规划成若干 zoom 窗口。
|
|||
|
|
|
|||
|
|
返回 `(windows, note)`；`windows is None` 表示**硬失败**（区域过大，已中止）。
|
|||
|
|
拿不到比例 / 绘图区像素时**退回单张**并如实告知（无法保证可读）。
|
|||
|
|
"""
|
|||
|
|
s = tiling.world_per_px(scale, config.TEXT_HEIGHT_MM, readable_px(purpose))
|
|||
|
|
eff = tiling.effective_px(view_px, config.IMAGE_MAX_EDGE)
|
|||
|
|
padded = tiling.expand_box(box, _EDGE_PAD)
|
|||
|
|
if padded is None or padded[2] <= padded[0] or padded[3] <= padded[1]:
|
|||
|
|
return None, "目标区域退化（宽或高为 0），无法放大"
|
|||
|
|
|
|||
|
|
if s <= 0 or eff[0] <= 0 or eff[1] <= 0:
|
|||
|
|
win = tiling.box_to_window(padded)
|
|||
|
|
return [win], "（拿不到出图比例或绘图区像素，已退回单张；文字可能仍读不清）"
|
|||
|
|
|
|||
|
|
cols, rows = tiling.tiles_for_box(padded, s * eff[0], s * eff[1], overlap)
|
|||
|
|
need = cols * rows
|
|||
|
|
if need > max_tiles:
|
|||
|
|
return None, (
|
|||
|
|
f"该区域按可读分辨率需 {need} 张截图，超过单次上限 {max_tiles}；"
|
|||
|
|
f"请缩小 target（例如用 label + cx + cy + span 指定更小范围）或分多次请求"
|
|||
|
|
)
|
|||
|
|
wins = tiling.subdivide(padded, cols, rows, overlap, frame_win=None)
|
|||
|
|
if not wins:
|
|||
|
|
return None, "目标区域退化，无法放大"
|
|||
|
|
return wins, f"按 {purpose} 判据切为 {cols}×{rows} = {need} 张"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def inspect_region(ctx, evidence, args, image_budget, state):
|
|||
|
|
"""执行一次放大取证，返回 `(content, image_budget)`。"""
|
|||
|
|
args = args if isinstance(args, dict) else {}
|
|||
|
|
purpose = str(args.get("purpose") or "text").strip().lower()
|
|||
|
|
if purpose not in ("text", "line"):
|
|||
|
|
purpose = "text"
|
|||
|
|
|
|||
|
|
box, note = resolve_target(evidence, args.get("target"), state)
|
|||
|
|
if box is None:
|
|||
|
|
return [_text(f"inspect_region 未执行：{note}")], image_budget
|
|||
|
|
|
|||
|
|
view_px = _view_px(evidence)
|
|||
|
|
scale = tiling.frame_scale(evidence.get("frame"))
|
|||
|
|
wins, plan_note = plan_tiles(
|
|||
|
|
box, scale, purpose, view_px,
|
|||
|
|
max_tiles=config.INSPECT_MAX_TILES,
|
|||
|
|
overlap=config.TILE_OVERLAP,
|
|||
|
|
margin=_ZOOM_MARGIN,
|
|||
|
|
)
|
|||
|
|
if wins is None:
|
|||
|
|
return [_text(f"inspect_region 未执行：{plan_note}")], image_budget
|
|||
|
|
|
|||
|
|
content = [_text(f"inspect_region：{note}；{plan_note}。")]
|
|||
|
|
taken = 0
|
|||
|
|
for i, win in enumerate(wins, 1):
|
|||
|
|
if image_budget[0] <= 0:
|
|||
|
|
content.append(_text(
|
|||
|
|
f"（截图预算已用完，剩余 {len(wins) - i + 1} 张未拍；"
|
|||
|
|
f"请基于已有信息给出结论，不要再请求截图。）"
|
|||
|
|
))
|
|||
|
|
break
|
|||
|
|
shot = zoom_and_shot(ctx, win, margin=_ZOOM_MARGIN)
|
|||
|
|
if not shot:
|
|||
|
|
continue
|
|||
|
|
image_budget[0] -= 1
|
|||
|
|
visible = tiling.visible_box(tiling.window_to_box(win), view_px, _ZOOM_MARGIN)
|
|||
|
|
label = state.register(visible)
|
|||
|
|
content.append(_text(
|
|||
|
|
f"（放大截图 {label}：可见世界范围 "
|
|||
|
|
f"x[{visible[0]:.1f}, {visible[2]:.1f}] y[{visible[1]:.1f}, {visible[3]:.1f}]）"
|
|||
|
|
))
|
|||
|
|
b64, mime = downsample_image_b64(shot[0], shot[1])
|
|||
|
|
content.append(LlmClient.image_content(b64, mime))
|
|||
|
|
taken += 1
|
|||
|
|
|
|||
|
|
if taken == 0:
|
|||
|
|
content.append(_text("（未能取得截图。）"))
|
|||
|
|
return content, image_budget
|