Создание мультимодальных визуальных агентов с Playwright и разметкой Set-of-Marks

import asyncio
import base64
from playwright.async_api import async_playwright
from openai import AsyncOpenAI

client = AsyncOpenAI()

SOM_INJECTION_SCRIPT = """
() => {
  let id = 1;
  const elements = document.querySelectorAll('button, a, input, select, textarea');
  elements.forEach(el => {
    const rect = el.getBoundingClientRect();
    if (rect.width > 10 && rect.height > 10 && window.getComputedStyle(el).visibility !== 'hidden') {
      const box = document.createElement('div');
      box.style.position = 'absolute';
      box.style.left = rect.left + window.scrollX + 'px';
      box.style.top = rect.top + window.scrollY + 'px';
      box.style.width = rect.width + 'px';
      box.style.height = rect.height + 'px';
      box.style.border = '2px solid #00f5a0';
      box.style.zIndex = '999999';
      box.innerHTML = `<span style="background:#00f5a0;color:#000;font-size:12px;font-weight:bold;padding:1px 4px;">${id}</span>`;
      document.body.appendChild(box);
      el.setAttribute('data-som-id', id);
      id++;
    }
  });
}
"""

async def run_vision_step(page):
    # 1. Размечаем элементы номерами
    await page.evaluate(SOM_INJECTION_SCRIPT)
    screenshot_bytes = await page.screenshot()
    base64_image = base64.b64encode(screenshot_bytes).decode('utf-8')

    # 2. Спрашиваем модель
    response = await client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": "Найди кнопку 'Войти' и назови только ее номер рамки."},
                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64_image}"}}
            ]
        }]
    )
    element_id = response.choices[0].message.content.strip()
    await page.click(f"[data-som-id='{element_id}']")
    print(f"Успешно кликнули на элемент #{element_id}")