Add Gemini computer use support

packages/browseros/chromium_patches/chrome/browser/resources/settings/nxtscape_page/models_data.ts

@@ -8,6 +8,7 @@ index 0000000000000..36638c1910a09
 +export interface ModelInfo {
 +  model_id: string;
 +  context_length: number;
++  requires_computer_use?: boolean;
 +}
 +
 +export interface ModelsData {
@@ -66,6 +67,7 @@ index 0000000000000..36638c1910a09
 +    { model_id: 'gemini-2.5-pro-preview-05-06', context_length: 1048576 },
 +    { model_id: 'gemini-2.5-pro-preview-06-05', context_length: 1048576 },
 +    { model_id: 'gemini-2.5-pro', context_length: 1048576 },
++    { model_id: 'gemini-2.5-computer-use-preview-10-2025', context_length: 1048576, requires_computer_use: true },
 +    { model_id: 'gemini-2.0-flash-exp', context_length: 1048576 },
 +    { model_id: 'gemini-2.0-flash', context_length: 1048576 },
 +    { model_id: 'gemini-2.0-flash-001', context_length: 1048576 },

packages/browseros/resources/files/ai_side_panel/BUILD.gn

@@ -0,0 +1,27 @@
+# Copyright 2024 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//ui/webui/resources/tools/generate_grd.gni")
+
+generate_grd("build") {
+  grd_prefix = "ai_side_panel"
+  out_grd = "$target_gen_dir/resources.grd"
+
+  input_files = [
+    "manifest.json",
+    "background.js",
+    "content.js",
+    "sidepanel.html",
+    "sidepanel.js",
+    "options.html",
+    "options.js",
+    "Readability.js",
+    "buildDomTree.js",
+    "assets/icon16.png",
+    "assets/icon48.png",
+    "assets/icon128.png",
+  ]
+
+  input_files_base_dir = rebase_path(".", "//")
+}

packages/browseros/resources/files/ai_side_panel/background_wrapper.js

@@ -0,0 +1,2 @@
+importScripts("computer_use_patch.js");
+importScripts("background.js");

packages/browseros/resources/files/ai_side_panel/computer_use_patch.js

@@ -0,0 +1,61 @@
+(() => {
+  const existing = globalThis.fetch;
+  if (!existing || existing.__browserosComputerUsePatched) return;
+
+  const originalFetch = existing.bind(globalThis);
+  const computerUseModels = new Set(["gemini-2.5-computer-use-preview-10-2025"]);
+
+  const normalizePayload = (payload) => {
+    if (!payload || typeof payload !== "object") return null;
+    if (!computerUseModels.has(payload.model)) return null;
+
+    const tools = Array.isArray(payload.tools) ? payload.tools.slice() : [];
+    const hasTool = tools.some((tool) => tool && typeof tool === "object" && "computer_use" in tool);
+
+    if (!hasTool) {
+      tools.push({ computer_use: { environment: "ENVIRONMENT_BROWSER" } });
+      payload.tools = tools;
+      return payload;
+    }
+
+    return payload;
+  };
+
+  const updateInit = (init) => {
+    if (!init || typeof init !== "object") return null;
+    if (!init.body) return null;
+
+    if (typeof init.body === "string") {
+      try {
+        const payload = JSON.parse(init.body);
+        const updated = normalizePayload(payload);
+        if (updated) {
+          return { ...init, body: JSON.stringify(updated) };
+        }
+      } catch {
+        return null;
+      }
+    } else if (typeof init.body === "object") {
+      const updated = normalizePayload(init.body);
+      if (updated) {
+        return { ...init, body: JSON.stringify(updated) };
+      }
+    }
+
+    return null;
+  };
+
+  const wrappedFetch = (...args) => {
+    const [input, init] = args;
+    const nextInit = updateInit(init);
+
+    if (nextInit) {
+      return originalFetch(input, nextInit);
+    }
+
+    return originalFetch(...args);
+  };
+
+  wrappedFetch.__browserosComputerUsePatched = true;
+  globalThis.fetch = wrappedFetch;
+})();

packages/browseros/resources/files/ai_side_panel/manifest.json

@@ -0,0 +1,63 @@
+{
+  "manifest_version": 3,
+  "name": "Agent",
+  "version": "40",
+  "description": "Agent",
+  "key": "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDC0C0AakL+XOcj+PMf24WqoKHA/hhypDQZUUd9sJr/c7cx3nLJS+kaJaB+cz8V3RhPZq01BFeByofLg6Hg6ZNVvBVn3wtGLcNPmuCwMmDiHetTV0oJsFOWs2VznP9py1ww5FFhU+Wmw1CWyLJqnwsIsLLCQyUXAZpVEOLJlOqX1wIDAQAB",
+  "permissions": [
+    "activeTab",
+    "storage",
+    "scripting",
+    "tabs",
+    "tabGroups",
+    "debugger",
+    "sidePanel",
+    "bookmarks",
+    "history",
+    "settingsPrivate",
+    "browserOS"
+  ],
+  "host_permissions": [
+    "<all_urls>"
+  ],
+  "background": {
+    "service_worker": "background_wrapper.js"
+  },
+  "action": {
+    "default_icon": {
+      "16": "assets/icon16.png",
+      "48": "assets/icon48.png",
+      "128": "assets/icon128.png"
+    }
+  },
+  "side_panel": {
+    "default_path": "sidepanel.html"
+  },
+  "commands": {
+    "toggle-panel": {
+      "suggested_key": {
+        "default": "Ctrl+E",
+        "mac": "Command+E"
+      },
+      "description": "Toggle the Nxtscape side panel"
+    }
+  },
+  "content_scripts": [
+    {
+      "matches": [
+        "<all_urls>"
+      ],
+      "js": [
+        "content.js"
+      ],
+      "run_at": "document_start",
+      "all_frames": true
+    }
+  ],
+  "icons": {
+    "16": "assets/icon16.png",
+    "48": "assets/icon48.png",
+    "128": "assets/icon128.png"
+  },
+  "component": true
+}

packages/browseros/resources/files/ai_side_panel/sidepanel.html

@@ -0,0 +1,22 @@
+<!doctype html><html lang="en" class="dark"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Nxtscape Side Panel</title><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600&display=swap" rel="stylesheet"><style>/* Reset default margins and add base font for side panel */
+    html, body {
+      width: 100%;
+      height: 100%;
+      margin: 0;
+      padding: 0;
+      font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Helvetica Neue', sans-serif;
+      background-color: #272728;
+      overflow: hidden;
+    }
+
+    #root {
+      width: 100%;
+      height: 100%;
+      display: flex;
+      flex-direction: column;
+    }
+
+    /* Side panel specific optimizations */
+    * {
+      box-sizing: border-box;
+    }</style><script defer="defer" src="sidepanel.js"></script></head><body><div id="root"></div><script src="sidepanel.js"></script></body></html>