feat(browser): B3 截图→上下文闭环 + B4 面板声明页(Y4)
- B3:截图落盘后**自动**并入本轮视图(能收图 ⇒ 真 image 部件挂 user 消息;收不了 ⇒ vision/OCR 文字;
大图先经 image-crop 裁切;取走即消费、每会话 ≤4 张、TTL 10min、跨会话隔离)
- B4:browser 面板**声明**(配置表单 + 运行状态/指纹自检/最近截图三张只读表 + 拍一张/重新自检)
+ 六个动作走 PLG-2 的通用动作端点(**不开新路由**)+ 保存事务(不可用则回滚 409)+ 热重载
- 读面**不启动浏览器**(计数型假 dialer 证明 0 次拨号);行数据不含路径/URL/内容
- 3 个变异(短路主路/行与列 key 脱钩/保存不回滚)
⚠️ **两行接线未落**(不在我文件面):wire_runtime.go 的 TransformContext + wire_frontend.go 的
wireBrowserPanel —— 落地前 B3/B4 只是「组件就绪」,跑着的后端里不存在(Y4 已写盯梢用例)
This commit is contained in:
1 parent
4213b268fe
commit
e0a6bca501
18 files changed
+3698
-72
No files matched your search
@@ -50,6 +50,10 @@ var _ sdk.Tool = (*Tool)(nil)
|
||||
type Tool struct {
|
||||
mgr Driver
|
||||
log *slog.Logger
|
||||
// outbox 是「本回合待送出的截图」附件箱(B3 闭环的浏览器侧,见 attach.go)。
|
||||
// 它一定非 nil(New 里默认建一个):nil 箱子会让「截图静默不进上下文」变成
|
||||
// 一个只能靠读代码才看得出的行为,而它恰恰是本片要修掉的缺口。
|
||||
outbox *ShotOutbox
|
||||
}
|
||||
|
||||
// Driver 是 Tool 需要的最小驱动面(*browsercap.Manager 实现它)。用接口而不是具体
|
||||
@@ -65,12 +69,18 @@ type Driver interface {
|
||||
// New 构造工具;mgr 为 nil 时工具**仍可构造**但每个动作都如实报「不可用」
|
||||
// (组合根的正确做法是压根不注册它,见 wire_browser.go)。
|
||||
func New(mgr Driver, opts ...Option) *Tool {
|
||||
t := &Tool{mgr: mgr, log: discardLog()}
|
||||
t := &Tool{mgr: mgr, log: discardLog(), outbox: NewShotOutbox()}
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(t)
|
||||
}
|
||||
}
|
||||
if t.outbox == nil {
|
||||
// WithShotOutbox(nil) 是「显式关掉附件」的写法,但工具面必须能解释自己:
|
||||
// 出一个可用的箱子、由附件状态字段如实报 disabled 由**不带箱**的路径给出,
|
||||
// 因此这里只兜住 nil(构造期 invariants 一处收紧,动作里就不必到处判空)。
|
||||
t.outbox = NewShotOutbox()
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
@@ -91,6 +101,46 @@ func WithLogger(l *slog.Logger) Option {
|
||||
return func(t *Tool) { t.log = logger.WithSource(l, logSource) }
|
||||
}
|
||||
|
||||
// WithShotOutbox 换用给定的截图附件箱(B3 闭环的浏览器侧,见 attach.go)。
|
||||
//
|
||||
// 传入 nil 当作「不换」:箱子是**必有**的(循环的装配钩子按工具名取回它,见
|
||||
// `cmd/cyrene/wire_browser_context.go`),把它做成可选只会让「截图到底进不进上下文」
|
||||
// 取决于装配顺序,而不是取决于设计。
|
||||
func WithShotOutbox(o *ShotOutbox) Option {
|
||||
if o == nil {
|
||||
return func(*Tool) {}
|
||||
}
|
||||
return func(t *Tool) { t.outbox = o }
|
||||
}
|
||||
|
||||
// ShotOutbox 返回本工具的附件箱。循环的装配钩子据此取走本回合待送出的截图
|
||||
// (取走即消费,见 ShotOutbox.Drain)。永不为 nil。
|
||||
func (t *Tool) ShotOutbox() *ShotOutbox {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
return t.outbox
|
||||
}
|
||||
|
||||
// State 返回面板要的**只读事实快照**(design/28 §8 B4;形状见
|
||||
// `internal/capability/browser/state.go`)。它**不启动浏览器、不发请求**:
|
||||
// 面板的只读区块连 viewer 都能看,让「看一眼」产生进程外副作用是越界。
|
||||
//
|
||||
// 驱动不支持这一面(假驱动 / 未来的别的实现)时返回零值快照 —— 面板据此如实显示
|
||||
// 「未知」,而不是编一组数字。
|
||||
func (t *Tool) State() browsercap.StateSnapshot {
|
||||
if t == nil || t.mgr == nil {
|
||||
return browsercap.StateSnapshot{}
|
||||
}
|
||||
src, ok := t.mgr.(interface {
|
||||
State() browsercap.StateSnapshot
|
||||
})
|
||||
if !ok {
|
||||
return browsercap.StateSnapshot{}
|
||||
}
|
||||
return src.State()
|
||||
}
|
||||
|
||||
// Name 实现 sdk.Tool。
|
||||
func (t *Tool) Name() string { return ToolName }
|
||||
|
||||
@@ -119,11 +169,12 @@ func (t *Tool) Description() string {
|
||||
return "Drive a real headful Chromium on this host and bring a page into the conversation. " +
|
||||
"Actions: " +
|
||||
"'open' navigates to an http(s) URL and returns the page's own title/url/readyState; " +
|
||||
"'screenshot' captures the current viewport, saves it to disk (0600) and returns its path — " +
|
||||
"pass that path to the vision or image-crop tool to actually read the page, which is how a " +
|
||||
"rendered chart/QR/login page becomes visible to a text-only model; " +
|
||||
"'screenshot' captures the current viewport, saves it to disk (0600) and attaches it to " +
|
||||
"THIS turn automatically — a sighted main model receives the image itself, a text-only " +
|
||||
"model receives a vision/OCR description — so you do not have to re-read it yourself; " +
|
||||
"'info' returns the browser fingerprint (headless, navigator.webdriver, screen, timezone, language) " +
|
||||
"for verifying that a headful session really is headful. " +
|
||||
"For a zoomed region of a large page use the image-crop tool on the returned path. " +
|
||||
"Rich interaction (click/type/wait/extract/network) is NOT implemented here on purpose: " +
|
||||
"configure a Playwright or Puppeteer MCP server and use its tools for that."
|
||||
}
|
||||
@@ -183,13 +234,19 @@ func (t *Tool) open(ctx context.Context, args map[string]any) (any, error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// screenshot 处理 action=screenshot:返回落盘路径与统计,「路径交 vision 读」这一步
|
||||
// 由模型按描述自行发起(本层不替它调模型,也不把图塞进上下文)。
|
||||
// screenshot 处理 action=screenshot:截图落盘(0600)后**把它挂进本回合的附件箱**
|
||||
// (B3 闭环:循环的下一次模型调用会把它作为 image 附件送出,见 attach.go),并返回
|
||||
// 落盘路径与统计。
|
||||
//
|
||||
// 为什么要「挂进箱子」而不是像 V5 那样只回路径:只回路径 = 把闭环交给模型自觉
|
||||
// (它得记得再调一次 vision),那不是闭环。现在两条路都自动:主模型能收图 ⇒ 它看到
|
||||
// 这张图;不能 ⇒ vision/OCR 工具的文字结果进上下文(design/28 §7 判据 2)。
|
||||
func (t *Tool) screenshot(ctx context.Context) (any, error) {
|
||||
shot, err := t.mgr.Screenshot(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status := t.outbox.PutForContext(ctx, shot)
|
||||
return map[string]any{
|
||||
"path": shot.Path,
|
||||
"bytes": shot.Bytes,
|
||||
@@ -199,10 +256,26 @@ func (t *Tool) screenshot(ctx context.Context) (any, error) {
|
||||
"height": shot.Height,
|
||||
"dimensionsKnown": shot.DimensionsKnown,
|
||||
"degraded": shot.Degraded,
|
||||
"hint": "Pass this path to the vision or image-crop tool to read the page content; the image is not inlined into the conversation.",
|
||||
"turnAttachment": status,
|
||||
"hint": shotHint(status),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// shotHint 是按附件状态给出的**如实**下一步提示(进模型上下文,因此措辞必须与事实一致:
|
||||
// queued 时说「已经作为图片附件送出」,其余档说「图没进上下文,按路径读」)。
|
||||
func shotHint(status string) string {
|
||||
switch status {
|
||||
case TurnAttachmentQueued:
|
||||
return "This screenshot is queued as an attachment for the current turn and will be " +
|
||||
"in your context on the next step (as the image itself, or as a vision/OCR description " +
|
||||
"if your model cannot receive images). Use image-crop on the path above to read a " +
|
||||
"region at full detail."
|
||||
default:
|
||||
return "This screenshot is NOT attached to the conversation (" + status + "). " +
|
||||
"Pass the path to the vision, OCR or image-crop tool to read the page content."
|
||||
}
|
||||
}
|
||||
|
||||
// info 处理 action=info。
|
||||
func (t *Tool) info(ctx context.Context) (any, error) {
|
||||
fp, err := t.mgr.Fingerprint(ctx)
|
||||
|
||||
Reference in new issue
Block a user