diff --git a/.gitignore b/.gitignore index 5ca059d..5bc1cb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules dist out +.worktrees .DS_Store .eslintcache *.log* diff --git a/docs/superpowers/plans/2026-06-03-safe-draft-mode.md b/docs/superpowers/plans/2026-06-03-safe-draft-mode.md new file mode 100644 index 0000000..d2fcf95 --- /dev/null +++ b/docs/superpowers/plans/2026-06-03-safe-draft-mode.md @@ -0,0 +1,562 @@ +# Safe Draft Mode Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a safe draft mode that fills the chat input with AI-generated text without sending, keeps auto-send available, and avoids repeated drafting when the chat content has not changed. + +**Architecture:** Keep the Provider contract unchanged and introduce `replyMode` as a first-class runtime setting. Split device actions into `fillDraft` and `sendMessage`, branch the session flow on `replyMode`, then expose the mode through the existing renderer settings and home control UI. Add a minimal behavior test harness around `GenericChannelSession` so draft and auto-send paths can be verified without spinning up Electron windows. + +**Tech Stack:** Electron, React 19, TypeScript, electron-store, robotjs, ts-node + +--- + +### Task 1: Add a runnable session behavior test harness + +**Files:** +- Modify: `package.json` +- Create: `scripts/test-generic-channel-session.ts` +- Modify: `src/core/device.ts:15-99` (interface additions referenced by the harness) +- Test: `scripts/test-generic-channel-session.ts` + +- [ ] **Step 1: Write the failing test harness** + +```ts +// scripts/test-generic-channel-session.ts +import assert from 'node:assert/strict' +import { GenericChannelSession, createInitialGenericChannelState } from '../src/core/generic-channel-session' +import type { DesktopDevice } from '../src/core/device' +import type { ChannelContext, ProviderEvent, SessionEvent } from '../src/core/session-types' + +class FakeDevice implements DesktopDevice { + actions: string[] = [] + setAppType(): void {} + setApiKey(): void {} + async measureLayout() { return { success: true } } + async screenshot() { return 'data:image/png;base64,fake' } + async hasUnreadMessage() { return { hasUnread: false } } + async isChatContactUnread() { return { isUnread: false } } + clearUnreadCache(): void {} + async setChatBaseline() { this.actions.push('setChatBaseline'); return true } + async hasChatAreaChanged() { return { hasDiff: false, hasBaseline: true } } + clearChatBaseline(): void {} + async fillDraft(text: string) { this.actions.push(`fillDraft:${text}`) } + async sendMessage(text: string) { this.actions.push(`sendMessage:${text}`) } + async activeUnreadByClick() {} + async clickUnreadContact() {} + async clickAt() {} +} + +async function runDraftScenario() { + const device = new FakeDevice() + const session = new GenericChannelSession(device) + const queued: SessionEvent[] = [] + const providerEvents: ProviderEvent[] = [{ type: 'reply_text', content: 'draft text' }] + const ctx: ChannelContext = { + appType: 'wechat', + state: createInitialGenericChannelState(), + host: { + enqueue: (event) => queued.push(event), + schedule: (event) => queued.push(event), + runProvider: async function* () { yield* providerEvents }, + log: () => {}, + isRunning: () => true, + stopSession: async () => {} + } + } + + await session.onEvent({ type: 'provider.reply_text', content: 'draft text' }, ctx) + + assert.deepEqual(device.actions, ['fillDraft:draft text', 'setChatBaseline']) + assert.equal(queued.at(-1)?.type, 'check_unread') +} + +runDraftScenario().catch((error) => { + console.error(error) + process.exit(1) +}) +``` + +- [ ] **Step 2: Add a package script for the harness** + +```json +{ + "scripts": { + "test:session": "ts-node --transpile-only scripts/test-generic-channel-session.ts" + } +} +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `npm run test:session` + +Expected: FAIL because `DesktopDevice` does not yet define `fillDraft`, and `GenericChannelSession` still sends replies through `sendMessage`. + +- [ ] **Step 4: Commit the red test setup** + +```bash +git add package.json scripts/test-generic-channel-session.ts +git commit -m "新增安全起草模式失败测试" +``` + +### Task 2: Add reply mode configuration and split device actions + +**Files:** +- Modify: `src/main/index.ts:40-127,372-381,485-501` +- Modify: `src/core/device.ts:15-99` +- Modify: `src/core/rpa/input-utils.ts:103-197` +- Modify: `src/core/rpa-device.ts:1-220` +- Modify: `src/core/box-select-device.ts:1-202` +- Modify: `src/core/mock-device.ts:1-71` +- Modify: `src/renderer/src/App.tsx:6-18,123-136` +- Test: `npm run test:session` + +- [ ] **Step 1: Extend settings and shared renderer types with `replyMode`** + +```ts +// src/main/index.ts +type ReplyMode = 'draft' | 'auto-send' + +interface AppSettings { + locale: 'zh' | 'en' + appType: AppType + replyMode: ReplyMode + vision: { apiKey: string } + chatProvider: { manifestUrl: string; installed: InstalledProviderInfo | null; config: Record } + defaultCaptureStrategy: CaptureStrategy + capture: Partial> +} + +const settingsStore = new StoreClass({ + name: 'settings', + defaults: { + locale: 'zh', + appType: 'wechat', + replyMode: 'draft', + vision: { apiKey: '' }, + chatProvider: { manifestUrl: '', installed: null, config: {} }, + defaultCaptureStrategy: 'auto', + capture: {} + } +}) +``` + +```ts +// src/renderer/src/App.tsx +type ReplyMode = 'draft' | 'auto-send' + +interface AppSettings { + locale: 'zh' | 'en' + appType: AppType + replyMode: ReplyMode + vision: { apiKey: string } + chatProvider: { + manifestUrl: string + installed: InstalledProviderInfo | null + config: Record + } + defaultCaptureStrategy: CaptureStrategy + capture: Partial> +} +``` + +- [ ] **Step 2: Add `fillDraft` to the device interface** + +```ts +// src/core/device.ts +export interface DesktopDevice { + setAppType(appType: AppType): void + setApiKey(apiKey: string): void + onSessionStart?(): Promise | void + onSessionStop?(): Promise | void + measureLayout(): Promise<{ success: boolean; error?: string }> + screenshot(): Promise + hasUnreadMessage(): Promise<{ hasUnread: boolean; chatEntranceArea?: { bbox: BBox; coordinates: [number, number] } }> + isChatContactUnread(): Promise<{ isUnread: boolean; firstContactCoords?: [number, number] }> + clearUnreadCache(): void + setChatBaseline(): Promise + hasChatAreaChanged(): Promise<{ hasDiff: boolean; hasBaseline: boolean }> + clearChatBaseline(): void + fillDraft(text: string): Promise + sendMessage(text: string): Promise + activeUnreadByClick(coordinates: [number, number]): Promise + clickUnreadContact(coordinates: [number, number]): Promise + clickAt(x: number, y: number): Promise +} +``` + +- [ ] **Step 3: Split low-level input actions into “paste only” and “paste + send”** + +```ts +// src/core/rpa/input-utils.ts +async function focusAndPasteByCoordsAction(x: number, y: number, text: string): Promise { + const robot = getRobot() + if (!robot) return false + + await humanLikeMove(x, y) + await randomDelayIn(100, 200) + robot.mouseClick('left') + await randomDelayIn(200, 300) + clipboard.writeText(text) + await randomDelayIn(50, 100) + + if (IS_MAC) { + robot.keyTap('v', ['command']) + } else { + robot.keyTap('v', ['control']) + } + + await randomDelayIn(300, 500) + return true +} + +export async function fillDraftByCoordsAction(x: number, y: number, text: string): Promise { + return focusAndPasteByCoordsAction(x, y, text) +} + +export async function sendReplyByCoordsAction(x: number, y: number, text: string): Promise { + const pasted = await focusAndPasteByCoordsAction(x, y, text) + if (!pasted) return false + const robot = getRobot() + if (!robot) return false + robot.keyTap('enter') + // keep existing platform-specific cleanup + return true +} +``` + +- [ ] **Step 4: Wire the new device method through all device implementations** + +```ts +// src/core/rpa-device.ts +import { fillDraftAction, sendReplyAction } from './rpa/input-utils' + +async fillDraft(text: string): Promise { + const success = await fillDraftAction(this.appType, text) + if (!success) throw new Error('写入草稿失败') +} + +async sendMessage(text: string): Promise { + const success = await sendReplyAction(this.appType, text) + if (!success) throw new Error('发送消息失败') +} +``` + +```ts +// src/core/box-select-device.ts +import { fillDraftByCoordsAction, sendReplyByCoordsAction } from './rpa/input-utils' + +async fillDraft(text: string): Promise { + const inputArea = getInputAreaFromCache(this.appType) + if (!inputArea) throw new Error('尚未测量输入框区域') + const [x, y] = inputArea.coordinates + const ok = await fillDraftByCoordsAction(x, y, text) + if (!ok) throw new Error('写入草稿失败') +} +``` + +```ts +// src/core/mock-device.ts +async fillDraft(text: string): Promise { + console.log(`[MockDevice] Drafted: ${text}`) +} +``` + +- [ ] **Step 5: Run test to verify interface and action wiring compile** + +Run: `npm run test:session` + +Expected: FAIL, but now the failure should be the session behavior still calling `sendMessage` instead of `fillDraft`. + +- [ ] **Step 6: Commit the device/config groundwork** + +```bash +git add package.json src/main/index.ts src/core/device.ts src/core/rpa/input-utils.ts src/core/rpa-device.ts src/core/box-select-device.ts src/core/mock-device.ts src/renderer/src/App.tsx +git commit -m "新增安全起草模式配置与设备动作" +``` + +### Task 3: Branch session behavior by reply mode and add draft log events + +**Files:** +- Modify: `src/core/session-types.ts` +- Modify: `src/core/generic-channel-session.ts` +- Modify: `src/core/runtime-host.ts` (if helper typings need expansion) +- Modify: `src/main/index.ts:129-131,485-501` +- Modify: `scripts/test-generic-channel-session.ts` +- Test: `scripts/test-generic-channel-session.ts` + +- [ ] **Step 1: Add reply mode and a `draft` log/event type to the session model** + +```ts +// src/core/session-types.ts +export type ReplyMode = 'draft' | 'auto-send' + +export type SessionEvent = + | { type: 'bootstrap' } + | { type: 'observe_chat' } + | { type: 'provider.thinking'; content: string } + | { type: 'provider.reply_text'; content: string } + | { type: 'provider.skip' } + | { type: 'provider.error'; error: string } + | { type: 'check_unread' } + | { type: 'wait_retry'; reason?: string; delayMs?: number } + +export interface ChannelContext { + appType: AppType + replyMode: ReplyMode + state: TState + host: RuntimeHostControls +} + +export interface RuntimeHostControls { + enqueue(event: SessionEvent): void + schedule(event: SessionEvent, delayMs: number): void + runProvider(input: ProviderInput): AsyncIterable + log(type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error', content: string): void + isRunning(): boolean + stopSession(reason?: string): Promise +} +``` + +- [ ] **Step 2: Feed `replyMode` into `RuntimeHost` and `GenericChannelSession`** + +```ts +// src/core/runtime-host.ts +interface RuntimeHostOptions { + appType: AppType + replyMode: ReplyMode + channel: ChannelSession + provider: ProviderAdapter + initialState: TState + onLog?: (type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error', content: string) => void +} + +this.context = { + appType: options.appType, + replyMode: options.replyMode, + state: options.initialState, + host: this.createControls() +} +``` + +```ts +// src/main/index.ts inside engine start/update paths +runtime = new RuntimeHost({ + appType: settings.appType, + replyMode: settings.replyMode, + channel: new GenericChannelSession(runtimeDevice), + provider, + initialState: createInitialGenericChannelState(), + onLog: (type, content) => { /* existing event bridge */ } +}) +``` + +- [ ] **Step 3: Branch `provider.reply_text` handling on `replyMode`** + +```ts +// src/core/generic-channel-session.ts +case 'provider.reply_text': { + if (ctx.replyMode === 'draft') { + await this.device.fillDraft(event.content) + ctx.host.log('draft', event.content) + } else { + await this.device.sendMessage(event.content) + ctx.host.log('reply', event.content) + } + + await this.device.setChatBaseline() + ctx.state.latestChatBaseline = Date.now() + ctx.host.enqueue({ type: 'check_unread' }) + break +} +``` + +- [ ] **Step 4: Expand the harness to verify both draft and auto-send paths** + +```ts +async function runAutoSendScenario() { + const device = new FakeDevice() + const session = new GenericChannelSession(device) + const ctx = createContext('auto-send') + + await session.onEvent({ type: 'provider.reply_text', content: 'final text' }, ctx) + + assert.deepEqual(device.actions, ['sendMessage:final text', 'setChatBaseline']) +} + +await runDraftScenario() +await runAutoSendScenario() +console.log('generic-channel-session behavior checks passed') +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `npm run test:session` + +Expected: PASS with `generic-channel-session behavior checks passed` + +- [ ] **Step 6: Commit the green session behavior** + +```bash +git add src/core/session-types.ts src/core/generic-channel-session.ts src/core/runtime-host.ts src/main/index.ts scripts/test-generic-channel-session.ts +git commit -m "实现安全起草模式会话分流" +``` + +### Task 4: Expose reply mode in the renderer and update copy + +**Files:** +- Modify: `src/renderer/src/App.tsx:6-10,123-136,264-381,523-583` +- Modify: `src/renderer/src/i18n.ts:16-31,76-90` +- Modify: `src/preload/index.ts` (only if renderer typing needs a narrow helper) +- Modify: `src/preload/index.d.ts` (only if helper types change) +- Test: `npm run typecheck` + +- [ ] **Step 1: Add the new log type and reply mode UI state** + +```ts +// src/renderer/src/App.tsx +interface LogEntry { + time: string + type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error' + content: string +} + +type ReplyMode = 'draft' | 'auto-send' +``` + +- [ ] **Step 2: Add a mode switch to the main control card** + +```tsx +function ReplyModeCard({ + replyMode, + running, + onChange +}: { + replyMode: ReplyMode + running: boolean + onChange: (mode: ReplyMode) => void +}) { + return ( +
+
运行模式
+
+ + +
+
+ {replyMode === 'draft' ? 'AI 只填入输入框,不会自动回车发送。' : 'AI 会自动发送生成的回复。'} +
+
+ ) +} +``` + +- [ ] **Step 3: Persist the mode through existing settings and engine update flows** + +```ts +const [replyMode, setReplyMode] = useState('draft') + +useEffect(() => { + void (async () => { + const settings = (await window.electron?.invoke('settings:getAll')) as AppSettings | undefined + setReplyMode(settings?.replyMode || 'draft') + })() +}, []) + +const handleReplyModeChange = useCallback(async (next: ReplyMode) => { + if (status === 'running') return + setReplyMode(next) + await window.electron?.invoke('settings:set', { replyMode: next }) + await window.electron?.invoke('engine:updateConfig', { + ...((await window.electron?.invoke('settings:getAll')) as AppSettings), + replyMode: next + }) +}, [status]) +``` + +- [ ] **Step 4: Update translation keys for the new draft log and mode copy** + +```ts +// src/renderer/src/i18n.ts +'control.log.draft': '起草', +'control.replyMode': '运行模式', +'control.replyMode.draft': '安全起草', +'control.replyMode.autoSend': '自动发送', +'control.replyMode.draftHint': 'AI 只填入输入框,不会自动回车发送。', +'control.replyMode.autoSendHint': 'AI 会自动发送生成的回复。' +``` + +- [ ] **Step 5: Run typecheck to verify renderer wiring** + +Run: `npm run typecheck` + +Expected: PASS + +- [ ] **Step 6: Commit the UI wiring** + +```bash +git add src/renderer/src/App.tsx src/renderer/src/i18n.ts src/preload/index.ts src/preload/index.d.ts +git commit -m "新增安全起草模式界面开关" +``` + +### Task 5: Run full verification and note any follow-up risk + +**Files:** +- Verify only: no new source files expected + +- [ ] **Step 1: Run the session behavior harness** + +Run: `npm run test:session` + +Expected: PASS with `generic-channel-session behavior checks passed` + +- [ ] **Step 2: Run static verification** + +Run: `npm run typecheck` + +Expected: PASS + +- [ ] **Step 3: Run production build verification** + +Run: `npm run build` + +Expected: PASS; a Vite chunk warning about `vision-utils.ts` is acceptable if it matches the current baseline and the build exits 0. + +- [ ] **Step 4: Inspect final diff** + +Run: `git status --short && git diff --stat` + +Expected: only the planned source files plus the pre-existing `package.json` / `package-lock.json` dependency fix if it remains intentionally part of the branch. + +- [ ] **Step 5: Commit verification-ready implementation** + +```bash +git add package.json package-lock.json src/main/index.ts src/core/device.ts src/core/rpa/input-utils.ts src/core/rpa-device.ts src/core/box-select-device.ts src/core/mock-device.ts src/core/session-types.ts src/core/generic-channel-session.ts src/core/runtime-host.ts src/renderer/src/App.tsx src/renderer/src/i18n.ts scripts/test-generic-channel-session.ts +git commit -m "实现安全起草模式" +``` + +## Spec coverage self-check + +- `replyMode` 默认值与配置持久化:Task 2 +- Provider 协议保持不变:Task 2 / Task 3 +- `fillDraft` 与 `sendMessage` 语义拆分:Task 2 +- session 按模式分流:Task 3 +- 日志区分起草和发送:Task 3 / Task 4 +- 首页模式切换:Task 4 +- draft 模式继续巡检并依赖 baseline 防重复起草:Task 3 +- 最小行为测试与构建验证:Task 1 / Task 3 / Task 5 + +## Placeholder scan + +- No `TODO`, `TBD`, or “similar to previous task” placeholders remain. +- All code-changing tasks include concrete snippets and exact commands. + +## Type consistency check + +- `ReplyMode` is consistently `draft | auto-send`. +- Device split uses `fillDraft(text)` and `sendMessage(text)` everywhere. +- Log types are consistently `thinking | reply | draft | skip | error`. diff --git a/docs/superpowers/specs/2026-06-03-safe-draft-mode-design.md b/docs/superpowers/specs/2026-06-03-safe-draft-mode-design.md new file mode 100644 index 0000000..1838eaa --- /dev/null +++ b/docs/superpowers/specs/2026-06-03-safe-draft-mode-design.md @@ -0,0 +1,227 @@ +# SightFlow 安全起草模式设计 + +日期:2026-06-03 + +## 背景 + +当前 SightFlow 在收到 Provider 返回的 `reply_text` 后,会直接调用设备层发送消息。这适合自动化演示,但不适合需要人工兜底的日常使用场景。目标是把默认运行路径调整为更安全的“AI 起草 + 人工确认发送”,同时保留原有自动发送模式,避免破坏已有能力。 + +## 目标 + +- 默认提供更安全的 `draft` 模式。 +- 在 `draft` 模式下,AI 仅将回复草稿写入聊天输入框,不自动发送。 +- 草稿写入后继续巡检其他未读会话。 +- 当前会话在聊天区内容未变化时不重复起草,避免覆盖用户已编辑的草稿。 +- 保留 `auto-send` 模式,并允许用户在首页快速切换。 + +## 非目标 + +- 不实现发送确认弹窗。 +- 不实现草稿历史列表或会话审核工作台。 +- 不修改 Provider 协议,不要求 Provider 感知起草模式。 +- 不尝试通过视觉方式强判断“用户是否已经手动发送成功”。 + +## 用户体验 + +### 运行模式 + +首页新增运行模式切换,提供两个选项: + +- `安全起草` +- `自动发送` + +默认选中 `安全起草`。运行中不允许切换模式,避免引擎状态和 UI 配置不一致。 + +### 安全起草模式 + +当 Provider 产出回复内容后: + +1. 应用将文本写入目标聊天输入框。 +2. 不按回车,不自动发送。 +3. 日志显示“已起草并填入输入框”。 +4. 当前会话进入“本轮已起草”状态。 +5. 引擎继续巡检其他未读会话。 + +### 自动发送模式 + +保留当前行为: + +1. 应用将文本写入并发送。 +2. 日志显示“已发送回复”。 +3. 继续后续未读巡检。 + +## 配置设计 + +在设置模型中新增配置项: + +```ts +type ReplyMode = 'draft' | 'auto-send' +``` + +配置名建议为 `replyMode`,存储在现有 settings 中,默认值为 `draft`。 + +该配置需要满足: + +- 主进程默认值存在。 +- 渲染层读取设置时可获得该值。 +- 启动引擎时传给 runtime。 +- 首页切换后立即持久化。 + +## 架构设计 + +### Provider 层 + +Provider 接口保持不变,仍只负责输出: + +- `thinking` +- `reply_text` +- `skip` +- `error` + +这样可以保持现有 Provider Hub 与外部 Provider 兼容。 + +### Session 层 + +`GenericChannelSession` 不再把 `reply_text` 直接等价为“发送成功”,而是根据 `replyMode` 分流: + +- `draft`:调用设备层 `fillDraft(text)`,记录日志为“已起草”,更新当前会话基线,并回到未读巡检。 +- `auto-send`:调用设备层 `sendMessage(text)`,记录日志为“已发送”,更新当前会话基线,并回到未读巡检。 + +Session 需要显式维护“当前会话是否已经在本轮起草”的语义,但不单独引入复杂持久化状态。第一版以聊天区 baseline 和现有轮询机制为主,避免额外数据库或缓存表。 + +### Device 层 + +现有设备接口需要拆分动作语义: + +- `fillDraft(text)`:写入输入框但不发送。 +- `sendMessage(text)`:保留现有发送语义。 + +`fillDraft(text)` 与 `sendMessage(text)` 应复用尽量多的底层输入逻辑,但对外保持明确语义,避免未来继续复用 `sendMessage` 做“伪起草”。 + +### UI 与日志 + +UI 需要新增: + +- 首页运行模式切换控件。 +- 与模式一致的说明文案。 + +日志展示需要区分: + +- `thinking`:正在分析、正在识别等。 +- `reply`:仅用于已发送。 +- `draft`:新增,表示草稿已写入。 +- `skip` +- `error` + +如果现有日志类型不适合区分“起草”和“发送”,应在事件和展示层增加正式的 `draft` 类型,而不是借用 `reply`。 + +## 数据流 + +### 安全起草模式 + +1. 检测到未读或当前聊天区变化。 +2. 截图并传给 Provider。 +3. Provider 返回 `reply_text`。 +4. Session 根据 `replyMode === 'draft'` 调用 `fillDraft(text)`。 +5. 写入成功后记录 `draft` 日志。 +6. 更新聊天区 baseline。 +7. 返回未读巡检。 + +### 自动发送模式 + +1. 检测到未读或当前聊天区变化。 +2. 截图并传给 Provider。 +3. Provider 返回 `reply_text`。 +4. Session 根据 `replyMode === 'auto-send'` 调用 `sendMessage(text)`。 +5. 发送成功后记录 `reply` 日志。 +6. 更新聊天区 baseline。 +7. 返回未读巡检。 + +## 去重策略 + +第一版不引入“会话 ID 去重表”,而采用现有聊天区变化检测机制。 + +在 `draft` 模式下,一旦草稿填入成功: + +- 立即更新当前聊天区 baseline。 +- 后续若当前聊天区无变化,则不重新观察并起草。 +- 只有当聊天区再次发生变化时,才允许再次生成草稿。 + +这样能满足以下目标: + +- 不覆盖同一轮已生成的草稿。 +- 用户手动编辑输入框时,不会因为系统重复起草而被打断。 +- 仍可继续切换其他未读会话。 + +已知限制: + +- 第一版不区分“用户手动发送成功”与“用户仅修改草稿未发送”。 +- 重新触发条件仅基于聊天区变化,而不是输入框状态。 + +这是有意为之的最小方案,优先降低误判复杂度。 + +## 错误处理 + +- `fillDraft(text)` 失败时,记录 `error`,本轮回到重试等待,不标记为已起草。 +- `sendMessage(text)` 失败时,保持现有错误处理逻辑。 +- 模式配置缺失时,按默认值 `draft` 处理,避免因缺配置退回不安全路径。 +- 运行中不允许切换模式;若 UI 尝试切换,应禁用控件而不是热切换 runtime。 + +## 测试策略 + +### 单元/行为测试 + +优先为 `GenericChannelSession` 增加最小行为测试: + +- `draft` 模式下,收到 `reply_text` 时调用 `fillDraft`,不调用 `sendMessage`。 +- `auto-send` 模式下,收到 `reply_text` 时调用 `sendMessage`,不调用 `fillDraft`。 +- `draft` 模式下,写入草稿后会继续进入 `check_unread` 流程。 +- `skip` 与 `error` 现有行为不回归。 + +### 回归验证 + +至少验证以下命令: + +- `npm run typecheck` +- `npm run build` + +如仓库已有可复用测试入口,应补充运行对应测试命令;若没有,则新增最小测试文件并运行该测试。 + +## 实现范围 + +本次实现仅覆盖: + +- `replyMode` 配置接入 +- 首页模式切换 +- Session 分流逻辑 +- Device 动作拆分 +- 日志类型和文案调整 +- 最小测试补充 + +本次不覆盖: + +- 确认弹窗 +- 草稿列表 +- 会话审核台 +- 输入框内容回读 +- 用户发送行为视觉检测 + +## 兼容性与迁移 + +- 老用户没有 `replyMode` 配置时,首次启动按默认值 `draft` 处理。 +- `auto-send` 仍然保留,因此不会阻断原有演示或测试流程。 +- Provider 无需升级。 + +## 风险 + +- 某些目标应用的“写入输入框但不发送”动作,可能需要额外确认现有输入实现是否天然支持不回车结束。 +- 仅靠聊天区变化做去重时,若聊天区变化非常轻微,可能触发重新起草;这属于后续观察项,不在第一版内过度优化。 +- UI 若未清晰展示当前模式,用户可能误判是否会自动发送,因此模式提示必须足够明确。 + +## 后续迭代方向 + +- 确认弹窗模式 +- 草稿审核面板 +- 已起草会话列表 +- 输入框草稿检测与覆盖保护 +- 基于会话标识的更强去重 diff --git a/package-lock.json b/package-lock.json index 6b56440..fb03e47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -141,7 +141,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -914,6 +913,7 @@ "dev": true, "license": "BSD-2-Clause", "optional": true, + "peer": true, "dependencies": { "cross-dirname": "^0.1.0", "debug": "^4.3.4", @@ -935,6 +935,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -951,6 +952,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -965,6 +967,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -1707,6 +1710,15 @@ "node-gyp-build": "^4.8.1" } }, + "node_modules/@hurdlegroup/robotjs/node_modules/node-addon-api": { + "version": "8.8.0", + "resolved": "https://registry.npmmirror.com/node-addon-api/-/node-addon-api-8.8.0.tgz", + "integrity": "sha512-c5Ko1fZJIJmzhFIkhRN76WTq+fC6tWnGy9CXA0fA+XygsWZmEwG8vmbkNqxMyoaa0Tin4djul49NzdVcJJcjeA==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -2861,7 +2873,6 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "dev": true, "license": "MIT", "optional": true, "engines": { @@ -3447,7 +3458,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -3470,7 +3480,6 @@ "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -3557,7 +3566,6 @@ "integrity": "sha512-/Zb/xaIDfxeJnvishjGdcR4jmr7S+bda8PKNhRGdljDM+elXhlvN0FyPSsMnLmJUrVG9aPO6dof80wjMawsASg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.58.2", "@typescript-eslint/types": "8.58.2", @@ -3839,7 +3847,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4504,7 +4511,6 @@ "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -5161,7 +5167,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -5824,7 +5829,8 @@ "integrity": "sha512-+R08/oI0nl3vfPcqftZRpytksBXDzOUveBq/NBVx0sUp1axwzPQrKinNx5yd5sxPu8j1wIy8AfnVQ+5eFdha6Q==", "dev": true, "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/cross-env": { "version": "10.1.0", @@ -6159,7 +6165,6 @@ "integrity": "sha512-glMJgnTreo8CFINujtAhCgN96QAqApDMZ8Vl1r8f0QT8QprvC1UCltV4CcWj20YoIyLZx6IUskaJZ0NV8fokcg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "app-builder-lib": "26.8.1", "builder-util": "26.8.1", @@ -6352,7 +6357,6 @@ "integrity": "sha512-B3TmzbUEeIvrhJ0QcoFp8/tgnVA3vsm0wkdYWzC22hsk9zTVqkzyrrz40cjd0nMTTIrGWxxfDO2tdQTCMe9Bjw==", "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@electron/get": "^2.0.0", "@types/node": "^22.7.7", @@ -6581,6 +6585,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@electron/asar": "^3.2.1", "debug": "^4.1.1", @@ -6601,6 +6606,7 @@ "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", @@ -6907,7 +6913,6 @@ "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6968,7 +6973,6 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", - "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -10004,6 +10008,7 @@ "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "minimist": "^1.2.6" }, @@ -10083,7 +10088,9 @@ "version": "1.7.2", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-1.7.2.tgz", "integrity": "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==", - "license": "MIT" + "dev": true, + "license": "MIT", + "optional": true }, "node_modules/node-api-version": { "version": "0.2.1", @@ -10774,7 +10781,6 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -10864,6 +10870,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "commander": "^9.4.0" }, @@ -10881,6 +10888,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "engines": { "node": "^12.20.0 || >=14" } @@ -10901,7 +10909,6 @@ "integrity": "sha512-8c3mgTe0ASwWAJK+78dpviD+A8EqhndQPUBpNUIPt6+xWlIigCwfN01lWr9MAede4uqXGTEKeQWTvzb3vjia0Q==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11027,7 +11034,6 @@ "integrity": "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -11254,6 +11260,7 @@ "deprecated": "Rimraf versions prior to v4 are no longer supported", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -12113,6 +12120,7 @@ "integrity": "sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "mkdirp": "^0.5.1", "rimraf": "~2.6.2" @@ -12448,7 +12456,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12646,7 +12653,6 @@ "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -13540,7 +13546,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index d6c9c87..0f59d63 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dev:test-screenshot": "electron-vite build && cross-env TEST_MODE=screenshot electron ./out/main/test-cli.js", "dev:test-reply": "electron-vite build && cross-env TEST_MODE=reply electron ./out/main/test-cli.js", "dev:test-switch": "electron-vite build && cross-env TEST_MODE=switch electron ./out/main/test-cli.js", + "test:session": "ts-node --transpile-only scripts/test-generic-channel-session.ts", "build": "npm run typecheck && electron-vite build", "postinstall": "patch-package && electron-builder install-app-deps", "build:unpack": "npm run build && electron-builder --dir", @@ -61,5 +62,10 @@ "ts-node": "^10.9.2", "typescript": "^5.9.3", "vite": "^7.2.6" + }, + "overrides": { + "@hurdlegroup/robotjs": { + "node-addon-api": "^8.8.0" + } } } diff --git a/scripts/test-generic-channel-session.ts b/scripts/test-generic-channel-session.ts new file mode 100644 index 0000000..42e05fc --- /dev/null +++ b/scripts/test-generic-channel-session.ts @@ -0,0 +1,115 @@ +import * as assert from 'node:assert/strict' +import { GenericChannelSession, createInitialGenericChannelState } from '../src/core/generic-channel-session' +import type { DesktopDevice } from '../src/core/device' +import type { ChannelContext, ReplyMode, SessionEvent } from '../src/core/session-types' + +class FakeDevice implements DesktopDevice { + actions: string[] = [] + + setAppType(): void {} + + setApiKey(): void {} + + async measureLayout(): Promise<{ success: boolean; error?: string }> { + return { success: true } + } + + async screenshot(): Promise { + return 'data:image/png;base64,fake' + } + + async hasUnreadMessage(): Promise<{ + hasUnread: boolean + chatEntranceArea?: { bbox: any; coordinates: [number, number] } + }> { + return { hasUnread: false } + } + + async isChatContactUnread(): Promise<{ + isUnread: boolean + firstContactCoords?: [number, number] + }> { + return { isUnread: false } + } + + clearUnreadCache(): void {} + + async setChatBaseline(): Promise { + this.actions.push('setChatBaseline') + return true + } + + async hasChatAreaChanged(): Promise<{ hasDiff: boolean; hasBaseline: boolean }> { + return { hasDiff: false, hasBaseline: true } + } + + clearChatBaseline(): void {} + + async sendMessage(text: string): Promise { + this.actions.push(`sendMessage:${text}`) + } + + async fillDraft(text: string): Promise { + this.actions.push(`fillDraft:${text}`) + } + + async activeUnreadByClick(): Promise {} + + async clickUnreadContact(): Promise {} + + async clickAt(): Promise {} +} + +async function runDraftScenario() { + const device = new FakeDevice() + const session = new GenericChannelSession(device) + const ctx = createContext('draft') + + await session.onEvent({ type: 'provider.reply_text', content: 'draft text' }, ctx) + + assert.deepEqual(device.actions, ['fillDraft:draft text', 'setChatBaseline']) + assert.equal(ctx.queued.at(-1)?.type, 'check_unread') +} + +async function runAutoSendScenario() { + const device = new FakeDevice() + const session = new GenericChannelSession(device) + const ctx = createContext('auto-send') + + await session.onEvent({ type: 'provider.reply_text', content: 'final text' }, ctx) + + assert.deepEqual(device.actions, ['sendMessage:final text', 'setChatBaseline']) + assert.equal(ctx.queued.at(-1)?.type, 'check_unread') +} + +function createContext(replyMode: ReplyMode): ChannelContext & { queued: SessionEvent[] } { + const queued: SessionEvent[] = [] + + const ctx: ChannelContext = { + appType: 'wechat', + replyMode, + state: createInitialGenericChannelState(), + host: { + enqueue: (event) => queued.push(event), + schedule: (event) => queued.push(event), + runProvider: async function* () { + return + }, + log: () => {}, + isRunning: () => true, + stopSession: async () => {} + } + } + + return { ...ctx, queued } +} + +runDraftScenario() + .then(async () => { + await runAutoSendScenario() + console.log('generic-channel-session behavior checks passed') + }) + .catch((error) => { + console.error(error) + process.exit(1) + }) diff --git a/src/core/box-select-device.ts b/src/core/box-select-device.ts index 6f3358c..3b440fb 100644 --- a/src/core/box-select-device.ts +++ b/src/core/box-select-device.ts @@ -26,6 +26,7 @@ import { activeUnreadByClickAction, clickUnreadContactAction, defaultClickPolicy, + fillDraftByCoordsAction, sendReplyByCoordsAction } from './rpa/input-utils' import { comparePngBuffers } from './rpa/image-compare' @@ -180,6 +181,14 @@ export class BoxSelectDevice implements DesktopDevice { // ── 动作层 ── + async fillDraft(text: string): Promise { + const inputArea = getInputAreaFromCache(this.appType) + if (!inputArea) throw new Error('尚未测量输入框区域') + const [x, y] = inputArea.coordinates + const ok = await fillDraftByCoordsAction(x, y, text) + if (!ok) throw new Error('写入草稿失败') + } + async sendMessage(text: string): Promise { const inputArea = getInputAreaFromCache(this.appType) if (!inputArea) throw new Error('尚未测量输入框区域') diff --git a/src/core/device.ts b/src/core/device.ts index 7b81fd0..8a45742 100644 --- a/src/core/device.ts +++ b/src/core/device.ts @@ -80,6 +80,9 @@ export interface DesktopDevice { // ── 动作层 ── + /** 填入草稿(clipboard paste,不发送) */ + fillDraft(text: string): Promise + /** 发送消息(clipboard paste + enter) */ sendMessage(text: string): Promise diff --git a/src/core/generic-channel-session.ts b/src/core/generic-channel-session.ts index 4a76aec..78aceee 100644 --- a/src/core/generic-channel-session.ts +++ b/src/core/generic-channel-session.ts @@ -73,8 +73,13 @@ export class GenericChannelSession implements ChannelSession { + console.log(`[MockDevice] Drafted: ${text}`) + } + async activeUnreadByClick(coordinates: [number, number]): Promise { console.log(`[MockDevice] activeUnreadByClick: (${coordinates[0]}, ${coordinates[1]})`) } diff --git a/src/core/rpa-device.ts b/src/core/rpa-device.ts index 7a112ab..2e0d8cc 100644 --- a/src/core/rpa-device.ts +++ b/src/core/rpa-device.ts @@ -9,7 +9,12 @@ import { AIClient } from './ai-client' import { AppType } from './rpa/types' import { BBox } from './rpa/vision-utils' import { captureChatMainArea } from './rpa/screenshot-utils' -import { sendReplyAction, activeUnreadByClickAction, clickUnreadContactAction } from './rpa/input-utils' +import { + fillDraftAction, + sendReplyAction, + activeUnreadByClickAction, + clickUnreadContactAction +} from './rpa/input-utils' import { hasUnreadMessage as hasUnreadMessageDetect, isChatContactUnread as isChatContactUnreadDetect @@ -228,6 +233,13 @@ export class RPADevice implements DesktopDevice { // ── 动作层 ── + async fillDraft(text: string): Promise { + const success = await fillDraftAction(this.appType, text) + if (!success) { + throw new Error('写入草稿失败') + } + } + async sendMessage(text: string): Promise { const success = await sendReplyAction(this.appType, text) if (!success) { diff --git a/src/core/rpa/input-utils.ts b/src/core/rpa/input-utils.ts index 0067e61..255297b 100644 --- a/src/core/rpa/input-utils.ts +++ b/src/core/rpa/input-utils.ts @@ -115,23 +115,10 @@ const getWeChatInputPosition = (bounds: any, scaleFactor: number) => { return { inputX: baseInputX + (Math.random() - 0.5) * 20, inputY: baseInputY - Math.random() * 5 } } -/** - * 业务原子 2 — 核心实现:按给定坐标发送消息(不依赖 VLM 缓存)。 - * `sendReplyAction`(VLM 路线)与 `BoxSelectDevice.sendMessage`(框选路线)共用此函数。 - * - * 1. humanLikeMove → 输入框焦点坐标 (x, y) - * 2. 隐式鼠标左键点击聚焦 - * 3. 剪贴板 + Cmd/Ctrl+V 粘贴 - * 4. Enter 发送 - */ -export async function sendReplyByCoordsAction( - x: number, - y: number, - text: string -): Promise { +async function focusAndPasteByCoordsAction(x: number, y: number, text: string): Promise { const robot = getRobot() if (!robot) { - console.error('[sendReplyByCoordsAction] RobotJS 缺失') + console.error('[focusAndPasteByCoordsAction] RobotJS 缺失') return false } @@ -153,6 +140,21 @@ export async function sendReplyByCoordsAction( await randomDelayIn(300, 500) + return true + } catch (err: any) { + console.error('[focusAndPasteByCoordsAction] Failed:', err) + return false + } +} + +async function finishSendAfterPaste(): Promise { + const robot = getRobot() + if (!robot) { + console.error('[finishSendAfterPaste] RobotJS 缺失') + return false + } + + try { robot.keyTap('enter') if (IS_WINDOWS) { @@ -169,9 +171,63 @@ export async function sendReplyByCoordsAction( return true } catch (err: any) { - console.error('[sendReplyByCoordsAction] Failed:', err) + console.error('[finishSendAfterPaste] Failed:', err) + return false + } +} + +export async function fillDraftByCoordsAction( + x: number, + y: number, + text: string +): Promise { + return focusAndPasteByCoordsAction(x, y, text) +} + +/** + * 业务原子 2 — 核心实现:按给定坐标发送消息(不依赖 VLM 缓存)。 + * `sendReplyAction`(VLM 路线)与 `BoxSelectDevice.sendMessage`(框选路线)共用此函数。 + */ +export async function sendReplyByCoordsAction( + x: number, + y: number, + text: string +): Promise { + const pasted = await focusAndPasteByCoordsAction(x, y, text) + if (!pasted) return false + return await finishSendAfterPaste() +} + +function resolveInputCoords( + appType: AppType, + bounds: { x: number; y: number; width: number; height: number }, + scaleFactor: number +): [number, number] { + const inputArea = getInputAreaFromCache(appType) + if (inputArea) { + const inputX = inputArea.coordinates[0] + (Math.random() - 0.5) * 10 + const inputY = inputArea.coordinates[1] + (Math.random() - 0.5) * 4 + return [inputX, inputY] + } + + const pos = getWeChatInputPosition(bounds, scaleFactor) + return [pos.inputX, pos.inputY] +} + +export async function fillDraftAction(appType: AppType, text: string): Promise { + const windowInfo = await getWindowInfo(appType, false) + if (!windowInfo || !windowInfo.bounds) { + console.error('[fillDraftAction] 无法获取窗口信息') return false } + + const [inputX, inputY] = resolveInputCoords( + appType, + windowInfo.bounds, + windowInfo.scaleFactor || 1 + ) + + return fillDraftByCoordsAction(inputX, inputY, text) } /** @@ -185,24 +241,11 @@ export async function sendReplyAction(appType: AppType, text: string): Promise { appType: AppType + replyMode: ReplyMode channel: ChannelSession provider: ProviderAdapter initialState: TState - onLog?: (type: 'thinking' | 'reply' | 'skip' | 'error', content: string) => void + onLog?: (type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error', content: string) => void } export class RuntimeHost { @@ -27,6 +29,7 @@ export class RuntimeHost { constructor(private readonly options: RuntimeHostOptions) { this.context = { appType: options.appType, + replyMode: options.replyMode, state: options.initialState, host: this.createControls() } @@ -77,6 +80,10 @@ export class RuntimeHost { this.context.appType = appType } + updateReplyMode(replyMode: ReplyMode): void { + this.context.replyMode = replyMode + } + private createControls(): RuntimeHostControls { return { enqueue: (event) => this.enqueue(event), @@ -125,7 +132,7 @@ export class RuntimeHost { } } - private log(type: 'thinking' | 'reply' | 'skip' | 'error', content: string): void { + private log(type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error', content: string): void { if (this.options.onLog) { this.options.onLog(type, content) } else { diff --git a/src/core/session-types.ts b/src/core/session-types.ts index 0d0e76e..de30226 100644 --- a/src/core/session-types.ts +++ b/src/core/session-types.ts @@ -1,5 +1,7 @@ import { AppType } from './rpa/types' +export type ReplyMode = 'draft' | 'auto-send' + export interface ProviderInput { screenshot: string appType: AppType @@ -31,13 +33,14 @@ export interface RuntimeHostControls { enqueue(event: SessionEvent): void schedule(event: SessionEvent, delayMs: number): void runProvider(input: ProviderInput): AsyncIterable - log(type: 'thinking' | 'reply' | 'skip' | 'error', content: string): void + log(type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error', content: string): void isRunning(): boolean stopSession(reason?: string): Promise } export interface ChannelContext { appType: AppType + replyMode: ReplyMode state: TState host: RuntimeHostControls } diff --git a/src/main/index.ts b/src/main/index.ts index 1669ad2..8a85d52 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -42,9 +42,12 @@ interface PerAppCapture { regions: BoxRegions | null } +type ReplyMode = 'draft' | 'auto-send' + interface AppSettings { locale: 'zh' | 'en' appType: AppType + replyMode: ReplyMode vision: { apiKey: string } @@ -115,6 +118,7 @@ const settingsStore = new StoreClass({ defaults: { locale: 'zh', appType: 'wechat', + replyMode: 'draft', vision: { apiKey: '' }, chatProvider: { manifestUrl: '', @@ -507,6 +511,7 @@ app.whenReady().then(async () => { } if (runtime) { runtime.updateAppType(settings.appType) + runtime.updateReplyMode(settings.replyMode) } return { success: true } }) @@ -681,7 +686,10 @@ async function startEngineCore(rawConfig?: any): Promise { } const mainWindow = BrowserWindow.getAllWindows().find((w) => !w.isDestroyed()) ?? null - const log = (type: 'thinking' | 'reply' | 'skip' | 'error', content: string): void => { + const log = ( + type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error', + content: string + ): void => { if (mainWindow && !mainWindow.isDestroyed()) { mainWindow.webContents.send('engine:log', { type, content }) } @@ -706,6 +714,7 @@ async function startEngineCore(rawConfig?: any): Promise { const channel = new GenericChannelSession(device) runtime = new RuntimeHost({ appType, + replyMode: settings.replyMode, channel, provider, initialState: createInitialGenericChannelState(), @@ -794,7 +803,7 @@ async function buildDevice( appType: AppType, settings: AppSettings, apiKey: string, - log: (type: 'thinking' | 'reply' | 'skip' | 'error', content: string) => void + log: (type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error', content: string) => void ): Promise<{ device: DesktopDevice; strategy: CaptureStrategy }> { const perApp = settings.capture[appType] ?? { strategy: 'auto' as CaptureStrategy, regions: null } const effective = resolveSettingsStrategy(appType, settings) @@ -937,6 +946,7 @@ function normalizeSettings(raw: any): AppSettings { return { locale: raw?.locale === 'en' ? 'en' : 'zh', appType: coerceAppType(raw?.appType), + replyMode: raw?.replyMode === 'auto-send' ? 'auto-send' : 'draft', vision: { apiKey: raw?.vision?.apiKey || oldApiKey || '' }, diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index ede3009..c96ae67 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -5,13 +5,14 @@ import './index.css' interface LogEntry { time: string - type: 'thinking' | 'reply' | 'skip' | 'error' + type: 'thinking' | 'reply' | 'draft' | 'skip' | 'error' content: string } type EngineStatus = 'idle' | 'running' | 'error' type SettingsSection = 'base' | 'agent' type AppType = 'wechat' | 'wework' | 'dingtalk' | 'lark' | 'slack' | 'telegram' | 'generic' +type ReplyMode = 'draft' | 'auto-send' type CaptureStrategy = 'auto' | 'vlm' | 'box-select' @@ -123,6 +124,7 @@ interface PerAppCapture { interface AppSettings { locale: 'zh' | 'en' appType: AppType + replyMode: ReplyMode vision: { apiKey: string } @@ -270,6 +272,7 @@ function ControlPanel({ // 首屏目标应用 + 框选状态:直接读 / 写 settings,让用户上手第一步就能完成。 const [appType, setAppType] = useState('wechat') + const [replyMode, setReplyMode] = useState('draft') const [regions, setRegions] = useState(null) const [openingWizard, setOpeningWizard] = useState(false) @@ -285,6 +288,7 @@ function ControlPanel({ | AppSettings | undefined const initial = settings?.appType || 'wechat' + setReplyMode(settings?.replyMode || 'draft') setAppType(initial) await reloadRegionsForApp(initial) })() @@ -335,6 +339,19 @@ function ControlPanel({ } }, [appType, status]) + const handleReplyModeChange = useCallback( + async (next: ReplyMode) => { + if (status === 'running') return + setReplyMode(next) + await window.electron?.invoke('settings:set', { replyMode: next }) + await window.electron?.invoke('engine:updateConfig', { + ...((await window.electron?.invoke('settings:getAll')) as AppSettings), + replyMode: next + }) + }, + [status] + ) + const addLog = useCallback((type: LogEntry['type'], content: string) => { const time = new Date().toLocaleTimeString('en-US', { hour12: false }) setLogs((prev) => [...prev.slice(-99), { time, type, content }]) @@ -385,6 +402,12 @@ function ControlPanel({ onOpenWizard={handleOpenWizard} /> + +
{t('control.log')}
@@ -407,6 +430,45 @@ function ControlPanel({ ) } +function ReplyModeCard({ + replyMode, + running, + onChange +}: { + replyMode: ReplyMode + running: boolean + onChange: (mode: ReplyMode) => void +}): React.JSX.Element { + return ( +
+
{t('control.replyMode')}
+
+ + +
+
+ {replyMode === 'draft' + ? t('control.replyMode.draftHint') + : t('control.replyMode.autoSendHint')} +
+
+ ) +} + interface TargetAppQuickCardProps { appType: AppType regions: BoxRegions | null diff --git a/src/renderer/src/i18n.ts b/src/renderer/src/i18n.ts index bf3bb3f..7a84b01 100644 --- a/src/renderer/src/i18n.ts +++ b/src/renderer/src/i18n.ts @@ -27,8 +27,14 @@ const translations = { 'control.log.empty': '引擎尚未启动', 'control.log.thinking': '思考', 'control.log.reply': '回复', + 'control.log.draft': '起草', 'control.log.skip': '跳过', 'control.log.error': '错误', + 'control.replyMode': '运行模式', + 'control.replyMode.draft': '安全起草', + 'control.replyMode.autoSend': '自动发送', + 'control.replyMode.draftHint': 'AI 只填入输入框,不会自动回车发送。', + 'control.replyMode.autoSendHint': 'AI 会自动发送生成的回复。', // Settings 'settings.vision': '视觉配置', @@ -86,8 +92,14 @@ const translations = { 'control.log.empty': 'Engine not started yet', 'control.log.thinking': 'Thinking', 'control.log.reply': 'Reply', + 'control.log.draft': 'Draft', 'control.log.skip': 'Skip', 'control.log.error': 'Error', + 'control.replyMode': 'Reply Mode', + 'control.replyMode.draft': 'Safe Draft', + 'control.replyMode.autoSend': 'Auto Send', + 'control.replyMode.draftHint': 'AI fills the input box but does not send automatically.', + 'control.replyMode.autoSendHint': 'AI sends the generated reply automatically.', 'settings.vision': 'Vision', 'settings.appType': 'App Type',