From b84c6f85dbaf5efc19510b66ab5047ae013c2138 Mon Sep 17 00:00:00 2001
From: xiaoski <xiaoski@qq.com>
Date: Sun, 24 May 2026 16:36:11 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=B5=8F=E8=A7=88=E5=99=A8?=
 =?UTF-8?q?=E5=B7=A5=E5=85=B7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/tools/browser.rs | 314 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 255 insertions(+), 59 deletions(-)

diff --git a/src/tools/browser.rs b/src/tools/browser.rs
index b1c25f2..d037370 100644
--- a/src/tools/browser.rs
+++ b/src/tools/browser.rs
@@ -5,7 +5,7 @@ use std::time::Duration;
 use anyhow::Context;
 use async_trait::async_trait;
 use base64::Engine;
-use fantoccini::actions::{InputSource, MouseActions, PointerAction};
+use fantoccini::actions::{InputSource, MouseActions, PointerAction, MOUSE_BUTTON_LEFT};
 use fantoccini::key::Key;
 use fantoccini::{Client, ClientBuilder, Locator};
 use serde::{Deserialize, Serialize};
@@ -70,11 +70,11 @@ pub enum BrowserAction {
         #[serde(default)]
         compact: bool,
         #[serde(default)]
-        depth: Option<u32>,
+        depth: Option<i64>,
     },
     Click { selector: String },
     Fill { selector: String, value: String },
-    Type { selector: String, text: String },
+    Type { selector: Option<String>, text: String },
     GetText { selector: String },
     GetTitle,
     GetUrl,
@@ -82,8 +82,9 @@ pub enum BrowserAction {
         #[serde(default)]
         path: Option<String>,
         #[serde(default)]
-        full_page: bool,
+        return_base64: bool,
     },
+    Focus { selector: String },
     Wait {
         #[serde(default)]
         selector: Option<String>,
@@ -94,6 +95,7 @@ pub enum BrowserAction {
     },
     Press { key: String },
     Hover { selector: String },
+    ClickAt { x: u32, y: u32 },
     Scroll {
         direction: String,
         #[serde(default)]
@@ -124,8 +126,7 @@ fn parse_browser_action(action_str: &str, args: &Value) -> anyhow::Result<Browse
                 .unwrap_or(true),
             depth: args
                 .get("depth")
-                .and_then(|v| v.as_u64())
-                .map(|d| u32::try_from(d).unwrap_or(u32::MAX)),
+                .and_then(|v| v.as_i64()),
         }),
         "click" => {
             let selector = args
@@ -154,13 +155,13 @@ fn parse_browser_action(action_str: &str, args: &Value) -> anyhow::Result<Browse
             let selector = args
                 .get("selector")
                 .and_then(|v| v.as_str())
-                .ok_or_else(|| anyhow::anyhow!("Missing 'selector' for type"))?;
+                .map(|s| s.to_string());
             let text = args
                 .get("text")
                 .and_then(|v| v.as_str())
                 .ok_or_else(|| anyhow::anyhow!("Missing 'text' for type"))?;
             Ok(BrowserAction::Type {
-                selector: selector.to_string(),
+                selector,
                 text: text.to_string(),
             })
         }
@@ -177,11 +178,20 @@ fn parse_browser_action(action_str: &str, args: &Value) -> anyhow::Result<Browse
         "get_url" => Ok(BrowserAction::GetUrl),
         "screenshot" => Ok(BrowserAction::Screenshot {
             path: args.get("path").and_then(|v| v.as_str()).map(String::from),
-            full_page: args
-                .get("full_page")
+            return_base64: args
+                .get("return_base64")
                 .and_then(Value::as_bool)
                 .unwrap_or(false),
         }),
+        "focus" => {
+            let selector = args
+                .get("selector")
+                .and_then(|v| v.as_str())
+                .ok_or_else(|| anyhow::anyhow!("Missing 'selector' for focus"))?;
+            Ok(BrowserAction::Focus {
+                selector: selector.to_string(),
+            })
+        }
         "wait" => Ok(BrowserAction::Wait {
             selector: args
                 .get("selector")
@@ -225,6 +235,17 @@ fn parse_browser_action(action_str: &str, args: &Value) -> anyhow::Result<Browse
             })
         }
         "close" => Ok(BrowserAction::Close),
+        "click_at" => {
+            let x = args
+                .get("x")
+                .and_then(|v| v.as_u64())
+                .ok_or_else(|| anyhow::anyhow!("Missing 'x' for click_at"))? as u32;
+            let y = args
+                .get("y")
+                .and_then(|v| v.as_u64())
+                .ok_or_else(|| anyhow::anyhow!("Missing 'y' for click_at"))? as u32;
+            Ok(BrowserAction::ClickAt { x, y })
+        }
         other => anyhow::bail!("Unsupported browser action: {}", other),
     }
 }
@@ -236,10 +257,18 @@ impl Tool for BrowserTool {
     }
 
     fn description(&self) -> &str {
-        "Automate browser interactions using WebDriver. \
-        First call open to navigate to a URL, then use other actions. \
-        Use snapshot to get an accessibility tree of the page. \
-        Selectors can be CSS, @e1 refs (from snapshot), text=..., or label=..."
+        "Automate browser interactions via WebDriver. \
+        Actions: open, snapshot, click, fill, type, get_text, get_title, \
+        get_url, screenshot, wait, press, hover, scroll, close, focus, click_at. \
+        Each session holds a single page; calling open again navigates \
+        the current page (does not open a new tab). \
+        Selectors: CSS, @e1 refs (from snapshot), text=... for text content, \
+        label=... for <label> association. \
+        Use focus to set focus on an element before typing or filling. \
+        After click_at focuses an element, use type without selector \
+        to type into the active element. \
+        Limitations: press supports single keys only (no Ctrl+/Shift+ combos); \
+        screenshot captures the visible viewport only."
     }
 
     fn parameters_schema(&self) -> Value {
@@ -252,61 +281,69 @@ impl Tool for BrowserTool {
                     "enum": [
                         "open", "snapshot", "click", "fill", "type",
                         "get_text", "get_title", "get_url", "screenshot",
-                        "wait", "press", "hover", "scroll", "close"
+                        "wait", "press", "hover", "scroll", "close", "focus", "click_at"
                     ]
                 },
                 "url": {
                     "type": "string",
-                    "description": "URL to navigate to (required for open)"
+                    "description": "(open) URL to navigate to"
                 },
                 "selector": {
                     "type": "string",
-                    "description": "CSS selector, @e1 ref, text=..., or label=..."
+                    "description": "(click/fill/type/get_text/hover/wait/focus) CSS selector, @e1 ref, text=..., or label=... Omit for type to target active element after click_at."
                 },
                 "value": {
                     "type": "string",
-                    "description": "Value to fill into a form field"
+                    "description": "(fill) Value to fill. On dynamic sites, try click or focus first to make element interactable."
                 },
                 "text": {
                     "type": "string",
-                    "description": "Text to type or wait for"
+                    "description": "(type/wait) Text to append or wait for. For type, selector optional — if omitted, types into active element."
                 },
                 "key": {
                     "type": "string",
-                    "description": "Key to press (Enter, Tab, Escape, Backspace, Delete, ArrowUp, ArrowDown, etc.)"
+                    "description": "(press) Key to press. Single keys only: Enter, Tab, Escape, Backspace, Delete, ArrowUp, ArrowDown, Space, etc."
                 },
                 "direction": {
                     "type": "string",
-                    "description": "Scroll direction",
+                    "description": "(scroll) Scroll direction",
                     "enum": ["up", "down", "left", "right"]
                 },
                 "pixels": {
                     "type": "integer",
-                    "description": "Pixels to scroll (default 600)"
+                    "description": "(scroll) Pixels to scroll (default 600)"
                 },
                 "ms": {
                     "type": "integer",
-                    "description": "Milliseconds to wait (for wait action)"
+                    "description": "(wait) Milliseconds to wait"
                 },
                 "path": {
                     "type": "string",
-                    "description": "File path to save screenshot to. If omitted, returns base64."
+                    "description": "(screenshot) File path to save screenshot. Omit to auto-save to /tmp/picobot_screenshot_<ts>.png"
                 },
-                "full_page": {
+                "return_base64": {
                     "type": "boolean",
-                    "description": "Take full-page screenshot"
+                    "description": "(screenshot) Also return image as base64 data URI"
                 },
                 "interactive_only": {
                     "type": "boolean",
-                    "description": "Only show interactive elements in snapshot (default true)"
+                    "description": "(snapshot) Only show interactive elements (default true)"
                 },
                 "compact": {
                     "type": "boolean",
-                    "description": "Compact snapshot output (default true)"
+                    "description": "(snapshot) Compact output mode (default true)"
                 },
                 "depth": {
                     "type": "integer",
-                    "description": "Max depth for snapshot traversal"
+                    "description": "(snapshot) Max DOM depth for traversal"
+                },
+                "x": {
+                    "type": "integer",
+                    "description": "(click_at) X coordinate in viewport pixels"
+                },
+                "y": {
+                    "type": "integer",
+                    "description": "(click_at) Y coordinate in viewport pixels"
                 }
             },
             "required": ["action"]
@@ -382,6 +419,12 @@ impl BrowserState {
                     .await?;
                 let client = self.active_client()?;
                 client.goto(&url).await?;
+                let _ = client
+                    .execute(
+                        "Object.defineProperty(navigator,'webdriver',{get:()=>undefined})",
+                        Vec::<Value>::new(),
+                    )
+                    .await;
                 let current = client.current_url().await?;
                 Ok(ToolResult {
                     success: true,
@@ -420,8 +463,32 @@ impl BrowserState {
             BrowserAction::Fill { selector, value } => {
                 let client = self.active_client()?;
                 let el = find_element(client, &selector).await?;
-                let _ = el.clear().await;
-                el.send_keys(&value).await?;
+                let fill_result = async {
+                    let _ = el.clear().await;
+                    tokio::time::sleep(Duration::from_millis(300)).await;
+                    el.send_keys(&value).await
+                }
+                .await;
+                match fill_result {
+                    Ok(()) => {}
+                    Err(ref e) if format!("{e}").to_lowercase().contains("not interactable") => {
+                        tracing::debug!("Fill via WebDriver failed, falling back to JS: {e}");
+                        let escaped = value
+                            .replace('\\', "\\\\")
+                            .replace('\'', "\\'")
+                            .replace('\n', "\\n");
+                        client
+                            .execute(
+                                &format!(
+                                    "arguments[0].value='{escaped}';arguments[0].dispatchEvent(new Event('input',{{bubbles:true}}));arguments[0].dispatchEvent(new Event('change',{{bubbles:true}}));"
+                                ),
+                                vec![serde_json::to_value(&el)?],
+                            )
+                            .await?;
+                    }
+                    Err(e) => return Err(e.into()),
+                }
+                tracing::debug!(action = "fill", output_len = value.len(), "Browser action completed");
                 Ok(ToolResult {
                     success: true,
                     output: format!("Filled {} with {}", selector, value),
@@ -430,13 +497,52 @@ impl BrowserState {
             }
             BrowserAction::Type { selector, text } => {
                 let client = self.active_client()?;
-                find_element(client, &selector)
-                    .await?
-                    .send_keys(&text)
-                    .await?;
+                let type_result = match selector {
+                    Some(ref sel) => {
+                        let el = find_element(client, sel).await?;
+                        el.send_keys(&text).await
+                    }
+                    None => {
+                        // Type into active element (useful after click_at)
+                        let el = client.active_element().await?;
+                        el.send_keys(&text).await
+                    }
+                };
+                match type_result {
+                    Ok(()) => {}
+                    Err(ref e) if format!("{e}").to_lowercase().contains("not interactable") => {
+                        tracing::debug!("Type via WebDriver failed, falling back to JS: {e}");
+                        let escaped = text
+                            .replace('\\', "\\\\")
+                            .replace('\'', "\\'")
+                            .replace('\n', "\\n");
+                        if let Some(ref sel) = selector {
+                            let el = find_element(client, sel).await?;
+                            client
+                                .execute(
+                                    &format!(
+                                        "arguments[0].value='{escaped}';arguments[0].dispatchEvent(new Event('input',{{bubbles:true}}));arguments[0].dispatchEvent(new Event('change',{{bubbles:true}}));"
+                                    ),
+                                    vec![serde_json::to_value(&el)?],
+                                )
+                                .await?;
+                        } else {
+                            client
+                                .execute(
+                                    &format!(
+                                        "var el=document.activeElement;el.value='{escaped}';el.dispatchEvent(new Event('input',{{bubbles:true}}));el.dispatchEvent(new Event('change',{{bubbles:true}}));"
+                                    ),
+                                    Vec::<Value>::new(),
+                                )
+                                .await?;
+                        }
+                    }
+                    Err(e) => return Err(e.into()),
+                }
+                let target = selector.as_deref().unwrap_or("activeElement");
                 Ok(ToolResult {
                     success: true,
-                    output: format!("Typed {} chars into {}", text.len(), selector),
+                    output: format!("Typed {} chars into {}", text.len(), target),
                     error: None,
                 })
             }
@@ -467,29 +573,50 @@ impl BrowserState {
                     error: None,
                 })
             }
-            BrowserAction::Screenshot { path, full_page } => {
+            BrowserAction::Screenshot { path, return_base64 } => {
                 let client = self.active_client()?;
                 let png = client.screenshot().await?;
-                let _ = full_page;
-
-                match path {
-                    Some(p) => {
-                        tokio::fs::write(&p, &png).await?;
-                        Ok(ToolResult {
-                            success: true,
-                            output: format!("Screenshot saved to {}", p),
-                            error: None,
-                        })
-                    }
-                    None => {
-                        let b64 = base64::engine::general_purpose::STANDARD.encode(&png);
-                        Ok(ToolResult {
-                            success: true,
-                            output: format!("data:image/png;base64,{}", b64),
-                            error: None,
-                        })
-                    }
+                let save_path = path.unwrap_or_else(|| {
+                    format!(
+                        "/tmp/picobot_screenshot_{}.png",
+                        std::time::SystemTime::now()
+                            .duration_since(std::time::UNIX_EPOCH)
+                            .unwrap_or_default()
+                            .as_secs()
+                    )
+                });
+                tokio::fs::write(&save_path, &png).await?;
+                if return_base64 {
+                    let b64 = base64::engine::general_purpose::STANDARD.encode(&png);
+                    tracing::debug!(action = "screenshot", output_len = b64.len(), "Browser action completed");
+                    return Ok(ToolResult {
+                        success: true,
+                        output: format!("Screenshot saved to {}. Base64: data:image/png;base64,{}", save_path, b64),
+                        error: None,
+                    });
                 }
+                tracing::debug!(action = "screenshot", output_len = save_path.len(), "Browser action completed");
+                Ok(ToolResult {
+                    success: true,
+                    output: format!("Screenshot saved to {}", save_path),
+                    error: None,
+                })
+            }
+            BrowserAction::Focus { selector } => {
+                let client = self.active_client()?;
+                let el = find_element(client, &selector).await?;
+                client
+                    .execute(
+                        "arguments[0].focus(); arguments[0].scrollIntoView({block:'center'});",
+                        vec![serde_json::to_value(el)?],
+                    )
+                    .await?;
+                tracing::debug!(action = "focus", output_len = selector.len(), "Browser action completed");
+                Ok(ToolResult {
+                    success: true,
+                    output: format!("Focused {}", selector),
+                    error: None,
+                })
             }
             BrowserAction::Wait {
                 selector,
@@ -559,6 +686,52 @@ impl BrowserState {
                     error: None,
                 })
             }
+            BrowserAction::ClickAt { x, y } => {
+                let client = self.active_client()?;
+                let actions = MouseActions::new("mouse".to_string())
+                    .then(PointerAction::MoveTo {
+                        duration: Some(Duration::from_millis(100)),
+                        x: x as f64,
+                        y: y as f64,
+                    })
+                    .then(PointerAction::Down {
+                        button: MOUSE_BUTTON_LEFT,
+                    })
+                    .then(PointerAction::Up {
+                        button: MOUSE_BUTTON_LEFT,
+                    });
+                client.perform_actions(actions).await?;
+                let _ = client.release_actions().await;
+                // Get info about the clicked element
+                let element_info: Value = client
+                    .execute(
+                        &format!(
+                            "(()=>{{var e=document.elementFromPoint({},{x});if(!e)return null;return{{tag:e.tagName.toLowerCase(),id:e.id||null,type:e.type||null,text:(e.value||e.placeholder||e.innerText||'').slice(0,80)}}}})()",
+                            x
+                        ),
+                        Vec::<Value>::new(),
+                    )
+                    .await
+                    .unwrap_or_default();
+                let desc = match element_info.as_object() {
+                    Some(info) => {
+                        let tag = info.get("tag").and_then(|v| v.as_str()).unwrap_or("?");
+                        let id = info.get("id").and_then(|v| v.as_str()).unwrap_or("");
+                        let el_type = info.get("type").and_then(|v| v.as_str()).unwrap_or("");
+                        let text = info.get("text").and_then(|v| v.as_str()).unwrap_or("");
+                        let id_str = if id.is_empty() { String::new() } else { format!("#{id}") };
+                        let type_str = if el_type.is_empty() { String::new() } else { format!("[type={el_type}]") };
+                        let text_str = if text.is_empty() { String::new() } else { format!(" ({text})") };
+                        format!("Clicked at ({x},{y}) on <{tag}{id_str}{type_str}>{text_str}")
+                    }
+                    None => format!("Clicked at ({}, {})", x, y),
+                };
+                Ok(ToolResult {
+                    success: true,
+                    output: desc,
+                    error: None,
+                })
+            }
             BrowserAction::Scroll { direction, pixels } => {
                 let client = self.active_client()?;
                 let amount = i64::from(pixels.unwrap_or(600));
@@ -671,7 +844,15 @@ impl BrowserState {
             args.push(Value::String("--disable-dev-shm-usage".to_string()));
         }
 
-        args.push(Value::String("--window-size=1280,720".to_string()));
+        args.push(Value::String("--window-size=1920,1080".to_string()));
+
+        // Anti-bot detection
+        args.push(Value::String(
+            "--disable-blink-features=AutomationControlled".to_string(),
+        ));
+        args.push(Value::String(
+            "--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36".to_string(),
+        ));
 
         if let Some(ref binary) = chrome_binary {
             chrome_options.insert("binary".to_string(), Value::String(binary.clone()));
@@ -681,6 +862,11 @@ impl BrowserState {
             chrome_options.insert("args".to_string(), Value::Array(args));
         }
 
+        chrome_options.insert(
+            "excludeSwitches".to_string(),
+            serde_json::json!(["enable-automation"]),
+        );
+
         capabilities.insert(
             "goog:chromeOptions".to_string(),
             Value::Object(chrome_options),
@@ -953,7 +1139,10 @@ fn webdriver_key(key: &str) -> String {
         "pageup" => Key::PageUp.to_string(),
         "pagedown" => Key::PageDown.to_string(),
         "space" => " ".to_string(),
-        other => other.to_string(),
+        other => {
+            tracing::warn!("Unrecognized key '{}', this will have no effect (press only supports single named keys)", other);
+            other.to_string()
+        }
     }
 }
 
@@ -980,7 +1169,9 @@ fn snapshot_script(interactive_only: bool, compact: bool, depth: Option<i64>) ->
     return rect.width > 0 && rect.height > 0;
   }};
 
-  const isInteractive = (el) => {{
+   const isInteractive = (el) => {{
+    if (el.disabled || el.readOnly) return false;
+    if (!isVisible(el)) return false;
     if (el.matches('a,button,input,select,textarea,summary,[role],*[tabindex]')) return true;
     return typeof el.onclick === 'function';
   }};
@@ -993,6 +1184,7 @@ fn snapshot_script(interactive_only: bool, compact: bool, depth: Option<i64>) ->
 
     const ref = '@e' + (++counter);
     el.setAttribute('data-zc-ref', ref);
+    const rect = el.getBoundingClientRect();
     nodes.push({{
       ref,
       depth,
@@ -1001,6 +1193,10 @@ fn snapshot_script(interactive_only: bool, compact: bool, depth: Option<i64>) ->
       role: el.getAttribute('role'),
       text,
       interactive,
+      x: Math.round(rect.x),
+      y: Math.round(rect.y),
+      w: Math.round(rect.width),
+      h: Math.round(rect.height),
     }});
   }};