如何让AI“看懂”网页？拆解 Browser-Use 的三大核心技术模块

├── Agent                                  # AI 代理│   ├── gif.py                            # 历史记录可视化│   ├── memory                            # 记忆模块│   │   ├── __init__.py│   │   ├── service.py│   │   └── views.py│   ├── message_manager                    # 消息管理│   │   ├── service.py│   │   ├── tests.py│   │   ├── utils.py│   │   └── views.py│   ├── playwright_script_generator.py│   ├── playwright_script_helpers.py│   ├── prompts.py                        # 提示词相关│   ├── service.py│   ├── system_prompt.md│   ├── tests.py│   └── views.py├── browser                                # 浏览器相关│   ├── __init__.py│   ├── browser.py│   ├── context.py│   ├── extensions.py│   ├── profile.py                        # 浏览器配置│   ├── session.py                        # 核心会话管理│   └── views.py├── cli.py├── controller                            # 工具Action相关│   ├── registry│   │   ├── service.py│   │   └── views.py│   ├── service.py│   └── views.py├── dom                                    # Dom 树解析&可交互元素处理│   ├── __init__.py│   ├── buildDomTree.js│   ├── clickable_element_processor│   │   └── service.py│   ├── history_tree_processor│   │   ├── service.py│   │   └── view.py│   ├── service.py│   ├── tests│   │   └── test_accessibility_playground.py│   └── views.py├── exceptions.py├── logging_config.py├── telemetry                              # 产品使用情况追踪，数据收集&分析模块│   ├── __init__.py│   ├── service.py│   └── views.py└── utils.py

// 函数入口function buildDomTree(node, parentIframe = null, isParentHighlighted = false){    // node: 当前要处理的 DOM 节点    // parentIframe: 父级 iframe（用于跨 iframe 处理）    // isParentHighlighted: 父节点是否已被高亮（状态传递）}
// 递归终止条件 - 防止无限递归if (!node || node.id === HIGHLIGHT_CONTAINER_ID ||     (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE)) {  if (debugMode) PERF_METRICS.nodeMetrics.skippedNodes++;  return null;  // 终止当前分支的递归}
// 根节点特殊处理if (node === document.body) {  const nodeData = {    tagName: 'body',    attributes: {},    xpath: '/body',    children: [],  };
  // 核心递归点1：处理 body 的所有子节点  for (const child of node.childNodes) {    const domElement = buildDomTree(child, parentIframe, false); // 🔄 递归调用    if (domElement) nodeData.children.push(domElement);  }
  const id = `${ID.current++}`;  DOM_HASH_MAP[id] = nodeData;  return id;}
// 核心递归点2：根据节点类型进行不同的递归处理if (node.tagName) {  const tagName = node.tagName.toLowerCase();
  // 场景1：iframe 递归处理  if (tagName === "iframe") {    try {      const iframeDoc = node.contentDocument || node.contentWindow?.document;      if (iframeDoc) {        for (const child of iframeDoc.childNodes) {          const domElement = buildDomTree(child, node, false); // 🔄 跨 iframe 递归          if (domElement) nodeData.children.push(domElement);        }      }    } catch (e) {      console.warn("Unable to access iframe:", e);    }  }    // 场景2：富文本编辑器递归处理  elseif (    node.isContentEditable ||    node.getAttribute("contenteditable") === "true" ||    node.id === "tinymce" ||    node.classList.contains("mce-content-body")  ) {    // 处理富文本内容 - 保持高亮状态传递    for (const child of node.childNodes) {      const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted); // 🔄 递归      if (domElement) nodeData.children.push(domElement);    }  }  // 场景3：常规元素递归处理  else {    // Shadow DOM 处理    if (node.shadowRoot) {      nodeData.shadowRoot = true;      for (const child of node.shadowRoot.childNodes) {        const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted); // 🔄 Shadow DOM 递归        if (domElement) nodeData.children.push(domElement);      }    }        // 最重要的递归处理：常规子节点    for (const child of node.childNodes) {      // 关键：高亮状态的递归传递      const passHighlightStatusToChild = nodeWasHighlighted || isParentHighlighted;      const domElement = buildDomTree(child, parentIframe, passHighlightStatusToChild); // 🔄 递归调用      if (domElement) nodeData.children.push(domElement);    }  }}

# service.py - _construct_dom_tree 方法@time_execution_async('--construct_dom_tree')async def _construct_dom_tree(self, eval_page: dict) -> tuple[DOMElementNode, SelectorMap]:    """从 JavaScript 结果构建 Python DOM 树 - 核心递归处理"""    js_node_map = eval_page['map']    js_root_id = eval_page['rootId']
    selector_map = {}    node_map = {}
    # 🔄 第一轮遍历：创建所有节点    for id, node_data in js_node_map.items():        node, children_ids = self._parse_node(node_data)        if node is None:            continue
        node_map[id] = node
        # 建立可交互元素的索引映射        if isinstance(node, DOMElementNode) and node.highlight_index is not None:            selector_map[node.highlight_index] = node
    # 🔄 第二轮遍历：建立父子关系（递归结构）    for id, node_data in js_node_map.items():        node = node_map.get(id)        if isinstance(node, DOMElementNode):            # 关键：递归建立父子关系            for child_id in node_data.get('children', []):                if child_id in node_map:                    child_node = node_map[child_id]                    child_node.parent = node      # 设置父节点                    node.children.append(child_node)  # 添加子节点
    return node_map[str(js_root_id)], selector_map

classClickableElementProcessor:    """可点击元素处理器"""        @staticmethod    def get_clickable_elements_hashes(dom_element: DOMElementNode) -> set[str]:        """获取所有可点击元素的哈希值集合"""        clickable_elements = ClickableElementProcessor.get_clickable_elements(dom_element)        return {ClickableElementProcessor.hash_dom_element(element) for element in clickable_elements}        @staticmethod    def hash_dom_element(dom_element: DOMElementNode) -> str:        """为 DOM 元素生成唯一哈希标识"""        # 1. 父级路径哈希        parent_branch_path = ClickableElementProcessor._get_parent_branch_path(dom_element)        branch_path_hash = ClickableElementProcessor._parent_branch_path_hash(parent_branch_path)                # 2. 属性哈希        attributes_hash = ClickableElementProcessor._attributes_hash(dom_element.attributes)                # 3. XPath 哈希        xpath_hash = ClickableElementProcessor._xpath_hash(dom_element.xpath)                # 4. 组合哈希        return ClickableElementProcessor._hash_string(f'{branch_path_hash}-{attributes_hash}-{xpath_hash}')

// 元素高亮 - 为 AI 提供视觉索引function highlightElement(element, index, parentIframe = null){  // 1. 创建高亮容器  let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);  if (!container) {    container = document.createElement("div");    container.id = HIGHLIGHT_CONTAINER_ID;    container.style.zIndex = "2147483640";  // 最高层级  }
  // 2. 为每个元素创建彩色边框和数字标签  const colors = ["#FF0000", "#00FF00", "#0000FF", "#FFA500"];  const baseColor = colors[index % colors.length];
  // 3. 多矩形支持 (处理复杂布局)  const rects = element.getClientRects();  for (const rect of rects) {    const overlay = document.createElement("div");    overlay.style.border = `2px solid ${baseColor}`;    overlay.style.backgroundColor = baseColor + "1A"; // 10% 透明度    // 设置位置和尺寸...  }}

[1]<header class='app-header' >  [2]<div class='logo' >    公司 LOGO  [3]<nav class='main-nav' >    [4]<a href='/dashboard' >控制台 />    [5]<a href='/projects' >项目管理 />    [6]<div class='user-menu' >      [7]<button class='user-avatar' >        [8]<img alt='用户头像' />      [9]<div class='dropdown-menu' >        [10]<a href='/profile' >个人资料 />        [11]<a href='/settings' >账户设置 />        [12]<button >退出登录 />[13]<main class='app-content' >  [14]<aside class='sidebar' >    [15]<ul class='nav-list' >      [16]<li >        [17]<a href='/tasks' >任务列表 />      [18]<li >        [19]<a href='/calendar' >日历视图 />  [20]<section class='content-area' >    [21]<div class='toolbar' >      [22]<button class='btn-primary' >新建任务 />      [23]<input type='search' placeholder='搜索任务' />      [24]<select name='filter' >        [25]<option value='all' >全部任务 />        [26]<option value='pending' >待处理 />    [27]<div class='task-list' >      任务列表内容      [28]<div class='task-item' >        [29]<input type='checkbox' />        完成网站设计        [30]<button class='edit-btn' >编辑 />        [31]<button class='delete-btn' >删除 />        *[32]*<button >新出现的按钮 />  # 用 * 标记新元素

# views.py - clickable_elements_to_string 方法def clickable_elements_to_string(self, include_attributes: list[str] | None = None) -> str:    """将 DOM 树递归转换为 LLM 可理解的文本格式"""    formatted_text = []        def process_node(node: DOMBaseNode, depth: int) -> None:        """📍 递归处理函数 - 深度优先遍历和格式化"""        next_depth = int(depth)        depth_str = depth * '\t'  # 缩进表示层级                if isinstance(node, DOMElementNode):            # 处理可交互元素            if node.highlight_index is not None:                next_depth += 1                # 格式化当前节点信息                text = node.get_all_text_till_next_clickable_element()                # ... 属性处理和格式化逻辑                formatted_text.append(formatted_line)                        # ⭐ 递归处理所有子节点            for child in node.children:                process_node(child, next_depth)  # 递归调用                elif isinstance(node, DOMTextNode):            # 处理文本节点            if (not node.has_parent_with_highlight_index() and                node.parent and node.parent.is_visible):                formatted_text.append(f'{depth_str}{node.text}')        process_node(self, 0)  # 从根节点开始递归    return '\n'.join(formatted_text)

MessageManager (最高层 - 业务逻辑)    ↓ 使用MessageManagerState (状态层)    ↓ 包含MessageHistory (历史管理层)    ↓ 包含ManagedMessage (消息包装层)    ↓ 包含MessageMetadata + BaseMessage (数据层)

Playwright├── Browser (浏览器进程) - 独立的操作系统进程│   ├── BrowserContext (浏览器上下文/会话) - 独立的用户会话│   │   ├── Page (页面/标签页) - 具体的网页标签│   │   │   ├── Frame (框架) - 页面中的iframe等│   │   │   │   └── ElementHandle (元素句柄)│   │   │   └── Worker (Web Worker)│   │   ├── Page (另一个标签页)│   │   └── ServiceWorker (服务工作者)│   ├── BrowserContext (另一个独立上下文)│   │   ├── Page│   │   └── Page│   └── BrowserContext (更多上下文...)├── Browser (另一个浏览器进程)│   └── ...└── Browser (更多浏览器进程...)

You are an AI agent designed to automate browser tasks. Your goal is to accomplish the ultimate task following the rules.
# Input FormatTaskPrevious stepsCurrent URLOpen TabsInteractive Elements[index]<type>text</type>- index: Numeric identifier for interaction- type: HTML element type(button, input, etc.)- text: Element description  Example:  [33]<div>User form</div>  \t*[35]*<button aria-label='Submit form'>Submit</button>- Only elements with numeric indexes in [] are interactive- (stacked) indentation (with \t) is important and means that the element is a(html) child of the element above(with a lower index)- Elements with \* are new elements that were added after the previous step(if url has not changed)# Response Rules1. RESPONSE FORMAT: You must ALWAYS respond with valid JSON in this exact format:{{"current_state": {{"evaluation_previous_goal": "Success|Failed|Unknown - Analyze the current elements and the image to check if the previous goals/actions are successful like intended by the task. Mention if something unexpected happened. Shortly state why/why not",   "memory": "Description of what has been done and what you need to remember. Be very specific. Count here ALWAYS how many times you have done something and how many remain. E.g. 0 out of 10 websites analyzed. Continue with abc and xyz",   "next_goal": "What needs to be done with the next immediate action"}},   "action":[{{"one_action_name": {{// action-specific parameter}}}}, // ... more actions in sequence]}}2. ACTIONS: You can specify multiple actions in the list to be executed in sequence. But always specify only one action name per item. Use maximum {max_actions} actions per sequence.Common action sequences:- Form filling: [{{"input_text": {{"index": 1, "text": "username"}}}}, {{"input_text": {{"index": 2, "text": "password"}}}}, {{"click_element": {{"index": 3}}}}]- Navigation and extraction: [{{"go_to_url": {{"url": "https://example.com"}}}}, {{"extract_content": {{"goal": "extract the names"}}}}]- Actions are executed in the given order- If the page changes after an action, the sequence is interrupted and you get the new state.- Only provide the action sequence until an action which changes the page state significantly.- Try to be efficient, e.g. fill forms at once, or chain actions where nothing changes on the page- only use multiple actions if it makes sense.3. ELEMENT INTERACTION:- Only use indexes of the interactive elements4. NAVIGATION & ERROR HANDLING:- If no suitable elements exist, use other functions to complete the task- If stuck, try alternative approaches - like going back to a previous page, new search, new tab etc.- Handle popups/cookies by accepting or closing them- Use scroll to find elements you are looking for- If you want to research something, open a new tab instead of using the current tab- If captcha pops up, try to solve it - elsetry a different approach- If the page is not fully loaded, use wait action5. TASK COMPLETION:- Use the done action as the last action as soon as the ultimate task is complete- Dont use "done" before you are done with everything the user asked you, except you reach the last step of max_steps.- If you reach your last step, use the done action even if the task is not fully finished. Provide all the information you have gathered so far. If the ultimate task is completely finished set success to true. If not everything the user asked for is completed set success in done to false!- If you have to do something repeatedly for example the task says for"each", or"for all", or"x times", count always inside "memory" how many times you have done it and how many remain. Don't stop until you have completed like the task asked you. Only call done after the last step.- Don't hallucinate actions- Make sure you include everything you found out for the ultimate task in the done text parameter. Do not just say you are done, but include the requested information of the task.6. VISUAL CONTEXT:- When an image is provided, use it to understand the page layout- Bounding boxes with labels on their top right corner correspond to element indexes7. Form filling:- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.8. Long tasks:- Keep track of the status and subresults in the memory.- You are provided with procedural memory summaries that condense previous task history (every N steps). Use these summaries to maintain context about completed actions, current progress, and next steps. The summaries appear in chronological order and contain key information about navigation history, findings, errors encountered, and current state. Refer to these summaries to avoid repeating actions and to ensure consistent progress toward the task goal.9. Extraction:- If your task is to find information - call extract_content on the specific pages to get and store the information.  Your responses must be always JSON with the specified format.