Add Gemini computer use support (Fixes #148)

Author: MagellaX

packages/browseros/chromium_patches/chrome/browser/resources/settings/nxtscape_page/models_data.ts

@@ -8,6 +8,7 @@ index 0000000000000..36638c1910a09
 +export interface ModelInfo {
 +  model_id: string;
 +  context_length: number;
++  requires_computer_use?: boolean;
 +}
 +
 +export interface ModelsData {
@@ -66,6 +67,7 @@ index 0000000000000..36638c1910a09
 +    { model_id: 'gemini-2.5-pro-preview-05-06', context_length: 1048576 },
 +    { model_id: 'gemini-2.5-pro-preview-06-05', context_length: 1048576 },
 +    { model_id: 'gemini-2.5-pro', context_length: 1048576 },
++    { model_id: 'gemini-2.5-computer-use-preview-10-2025', context_length: 1048576, requires_computer_use: true },
 +    { model_id: 'gemini-2.0-flash-exp', context_length: 1048576 },
 +    { model_id: 'gemini-2.0-flash', context_length: 1048576 },
 +    { model_id: 'gemini-2.0-flash-001', context_length: 1048576 },

packages/browseros/resources/files/ai_side_panel/background_wrapper.js

@@ -0,0 +1,2 @@
+importScripts("computer_use_patch.js");
+importScripts("background.js");

packages/browseros/resources/files/ai_side_panel/computer_use_patch.js

@@ -0,0 +1,61 @@
+(() => {
+  const existing = globalThis.fetch;
+  if (!existing || existing.__browserosComputerUsePatched) return;
+
+  const originalFetch = existing.bind(globalThis);
+  const computerUseModels = new Set(["gemini-2.5-computer-use-preview-10-2025"]);
+
+  const normalizePayload = (payload) => {
+    if (!payload || typeof payload !== "object") return null;
+    if (!computerUseModels.has(payload.model)) return null;
+
+    const tools = Array.isArray(payload.tools) ? payload.tools.slice() : [];
+    const hasTool = tools.some((tool) => tool && typeof tool === "object" && "computer_use" in tool);
+
+    if (!hasTool) {
+      tools.push({ computer_use: { environment: "ENVIRONMENT_BROWSER" } });
+      payload.tools = tools;
+      return payload;
+    }
+
+    return payload;
+  };
+
+  const updateInit = (init) => {
+    if (!init || typeof init !== "object") return null;
+    if (!init.body) return null;
+
+    if (typeof init.body === "string") {
+      try {
+        const payload = JSON.parse(init.body);
+        const updated = normalizePayload(payload);
+        if (updated) {
+          return { ...init, body: JSON.stringify(updated) };
+        }
+      } catch {
+        return null;
+      }
+    } else if (typeof init.body === "object") {
+      const updated = normalizePayload(init.body);
+      if (updated) {
+        return { ...init, body: JSON.stringify(updated) };
+      }
+    }
+
+    return null;
+  };
+
+  const wrappedFetch = (...args) => {
+    const [input, init] = args;
+    const nextInit = updateInit(init);
+
+    if (nextInit) {
+      return originalFetch(input, nextInit);
+    }
+
+    return originalFetch(...args);
+  };
+
+  wrappedFetch.__browserosComputerUsePatched = true;
+  globalThis.fetch = wrappedFetch;
+})();

packages/browseros/resources/files/ai_side_panel/manifest.json

@@ -21,7 +21,7 @@
     "<all_urls>"
   ],
   "background": {
-    "service_worker": "background.js"
+    "service_worker": "background_wrapper.js"
   },
   "action": {
     "default_icon": {
@@ -60,4 +60,4 @@
     "128": "assets/icon128.png"
   },
   "component": true
-}
+}