diff --git a/src/docs.json b/src/docs.json index 90b2014630..a665e90409 100644 --- a/src/docs.json +++ b/src/docs.json @@ -629,14 +629,16 @@ "pages": [ "oss/javascript/integrations/providers/openai", "oss/javascript/integrations/chat/openai", - "oss/javascript/integrations/text_embedding/openai" + "oss/javascript/integrations/text_embedding/openai", + "oss/javascript/integrations/tools/openai" ] }, { "group": "Anthropic", "pages": [ "oss/javascript/integrations/providers/anthropic", - "oss/javascript/integrations/chat/anthropic" + "oss/javascript/integrations/chat/anthropic", + "oss/javascript/integrations/tools/anthropic" ] }, { diff --git a/src/oss/javascript/integrations/tools/anthropic.mdx b/src/oss/javascript/integrations/tools/anthropic.mdx new file mode 100644 index 0000000000..854e18bafd --- /dev/null +++ b/src/oss/javascript/integrations/tools/anthropic.mdx @@ -0,0 +1,614 @@ +--- +title: Tools +--- + +The `@langchain/anthropic` package provides LangChain-compatible wrappers for Anthropic's built-in tools. These tools can be bound to `ChatAnthropic` using `bindTools()` or @[`createAgent`]. + +### Memory Tool + +The memory tool (`memory_20250818`) enables Claude to store and retrieve information across conversations through a memory file directory. Claude can create, read, update, and delete files that persist between sessions, allowing it to build knowledge over time without keeping everything in the context window. + +```typescript +import { ChatAnthropic, tools } from "@langchain/anthropic"; + +// Create a simple in-memory file store (or use your own persistence layer) +const files = new Map(); + +const memory = tools.memory_20250818({ + execute: async (command) => { + switch (command.command) { + case "view": + if (!command.path || command.path === "/") { + return Array.from(files.keys()).join("\n") || "Directory is empty."; + } + return ( + files.get(command.path) ?? `Error: File not found: ${command.path}` + ); + case "create": + files.set(command.path!, command.file_text ?? ""); + return `Successfully created file: ${command.path}`; + case "str_replace": + const content = files.get(command.path!); + if (content && command.old_str) { + files.set( + command.path!, + content.replace(command.old_str, command.new_str ?? "") + ); + } + return `Successfully replaced text in: ${command.path}`; + case "delete": + files.delete(command.path!); + return `Successfully deleted: ${command.path}`; + // Handle other commands: insert, rename + default: + return `Unknown command`; + } + }, +}); + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +const llmWithMemory = llm.bindTools([memory]); + +const response = await llmWithMemory.invoke( + "Remember that my favorite programming language is TypeScript" +); +``` + +For more information, see [Anthropic's Memory Tool documentation](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/memory-tool). + +### Web Search Tool + +The web search tool (`webSearch_20250305`) gives Claude direct access to real-time web content, allowing it to answer questions with up-to-date information beyond its knowledge cutoff. Claude automatically cites sources from search results as part of its answer. + +```typescript +import { ChatAnthropic, tools } from "@langchain/anthropic"; + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +// Basic usage +const response = await llm.invoke("What is the weather in NYC?", { + tools: [tools.webSearch_20250305()], +}); +``` + +The web search tool supports several configuration options: + +```typescript +const response = await llm.invoke("Latest news about AI?", { + tools: [ + tools.webSearch_20250305({ + // Maximum number of times the tool can be used in the API request + maxUses: 5, + // Only include results from these domains + allowedDomains: ["reuters.com", "bbc.com"], + // Or block specific domains (cannot be used with allowedDomains) + // blockedDomains: ["example.com"], + // Provide user location for more relevant results + userLocation: { + type: "approximate", + city: "San Francisco", + region: "California", + country: "US", + timezone: "America/Los_Angeles", + }, + }), + ], +}); +``` + +For more information, see [Anthropic's Web Search Tool documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/web-search-tool). + +### Web Fetch Tool + +The web fetch tool (`webFetch_20250910`) allows Claude to retrieve full content from specified web pages and PDF documents. Claude can only fetch URLs that have been explicitly provided by the user or that come from previous web search or web fetch results. + +> **⚠️ Security Warning:** Enabling the web fetch tool in environments where Claude processes untrusted input alongside sensitive data poses data exfiltration risks. We recommend only using this tool in trusted environments or when handling non-sensitive data. + +```typescript +import { ChatAnthropic, tools } from "@langchain/anthropic"; + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +// Basic usage - fetch content from a URL +const response = await llm.invoke( + "Please analyze the content at https://example.com/article", + { tools: [tools.webFetch_20250910()] } +); +``` + +The web fetch tool supports several configuration options: + +```typescript +const response = await llm.invoke( + "Summarize this research paper: https://arxiv.org/abs/2024.12345", + { + tools: [ + tools.webFetch_20250910({ + // Maximum number of times the tool can be used in the API request + maxUses: 5, + // Only fetch from these domains + allowedDomains: ["arxiv.org", "example.com"], + // Or block specific domains (cannot be used with allowedDomains) + // blockedDomains: ["example.com"], + // Enable citations for fetched content (optional, unlike web search) + citations: { enabled: true }, + // Maximum content length in tokens (helps control token usage) + maxContentTokens: 50000, + }), + ], + } +); +``` + +You can combine web fetch with web search for comprehensive information gathering: + +```typescript +import { tools } from "@langchain/anthropic"; + +const response = await llm.invoke( + "Find recent articles about quantum computing and analyze the most relevant one", + { + tools: [ + tools.webSearch_20250305({ maxUses: 3 }), + tools.webFetch_20250910({ maxUses: 5, citations: { enabled: true } }), + ], + } +); +``` + +For more information, see [Anthropic's Web Fetch Tool documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/web-fetch-tool). + +### Tool Search Tools + +The tool search tools enable Claude to work with hundreds or thousands of tools by dynamically discovering and loading them on-demand. This is useful when you have a large number of tools but don't want to load them all into the context window at once. + +There are two variants: + +- **`toolSearchRegex_20251119`** - Claude constructs regex patterns (using Python's `re.search()` syntax) to search for tools +- **`toolSearchBM25_20251119`** - Claude uses natural language queries to search for tools using the BM25 algorithm + +```typescript +import { ChatAnthropic, tools } from "@langchain/anthropic"; +import { tool } from "langchain"; +import { z } from "zod"; + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +// Create tools with defer_loading to make them discoverable via search +const getWeather = tool( + async (input: { location: string }) => { + return `Weather in ${input.location}: Sunny, 72°F`; + }, + { + name: "get_weather", + description: "Get the weather at a specific location", + schema: z.object({ + location: z.string(), + }), + extras: { defer_loading: true }, + } +); + +const getNews = tool( + async (input: { topic: string }) => { + return `Latest news about ${input.topic}...`; + }, + { + name: "get_news", + description: "Get the latest news about a topic", + schema: z.object({ + topic: z.string(), + }), + extras: { defer_loading: true }, + } +); + +// Claude will search and discover tools as needed +const response = await llm.invoke("What is the weather in San Francisco?", { + tools: [tools.toolSearchRegex_20251119(), getWeather, getNews], +}); +``` + +Using the BM25 variant for natural language search: + +```typescript +import { tools } from "@langchain/anthropic"; + +const response = await llm.invoke("What is the weather in San Francisco?", { + tools: [tools.toolSearchBM25_20251119(), getWeather, getNews], +}); +``` + +For more information, see [Anthropic's Tool Search documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search-tool). + +### Text Editor Tool + +The text editor tool (`textEditor_20250728`) enables Claude to view and modify text files, helping debug, fix, and improve code or other text documents. Claude can directly interact with files, providing hands-on assistance rather than just suggesting changes. + +Available commands: + +- `view` - Examine file contents or list directory contents +- `str_replace` - Replace specific text in a file +- `create` - Create a new file with specified content +- `insert` - Insert text at a specific line number + +```typescript +import fs from "node:fs"; +import { ChatAnthropic, tools } from "@langchain/anthropic"; + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +const textEditor = tools.textEditor_20250728({ + async execute(args) { + switch (args.command) { + case "view": + const content = fs.readFileSync(args.path, "utf-8"); + // Return with line numbers for Claude to reference + return content + .split("\n") + .map((line, i) => `${i + 1}: ${line}`) + .join("\n"); + case "str_replace": + let fileContent = fs.readFileSync(args.path, "utf-8"); + fileContent = fileContent.replace(args.old_str, args.new_str); + fs.writeFileSync(args.path, fileContent); + return "Successfully replaced text."; + case "create": + fs.writeFileSync(args.path, args.file_text); + return `Successfully created file: ${args.path}`; + case "insert": + const lines = fs.readFileSync(args.path, "utf-8").split("\n"); + lines.splice(args.insert_line, 0, args.new_str); + fs.writeFileSync(args.path, lines.join("\n")); + return `Successfully inserted text at line ${args.insert_line}`; + default: + return "Unknown command"; + } + }, + // Optional: limit file content length when viewing + maxCharacters: 10000, +}); + +const llmWithEditor = llm.bindTools([textEditor]); + +const response = await llmWithEditor.invoke( + "There's a syntax error in my primes.py file. Can you help me fix it?" +); +``` + +For more information, see [Anthropic's Text Editor Tool documentation](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/text-editor-tool). + +### Computer Use Tool + +The computer use tools enable Claude to interact with desktop environments through screenshot capture, mouse control, and keyboard input for autonomous desktop interaction. + +> **⚠️ Security Warning:** Computer use is a beta feature with unique risks. Use a dedicated virtual machine or container with minimal privileges. Avoid giving access to sensitive data. + +There are two variants: + +- **`computer_20251124`** - For Claude Opus 4.5 (includes zoom capability) +- **`computer_20250124`** - For Claude 4 and Claude 3.7 models + +Available actions: + +- `screenshot` - Capture the current screen +- `left_click`, `right_click`, `middle_click` - Mouse clicks at coordinates +- `double_click`, `triple_click` - Multi-click actions +- `left_click_drag` - Click and drag operations +- `left_mouse_down`, `left_mouse_up` - Granular mouse control +- `scroll` - Scroll the screen +- `type` - Type text +- `key` - Press keyboard keys/shortcuts +- `mouse_move` - Move the cursor +- `hold_key` - Hold a key while performing other actions +- `wait` - Wait for a specified duration +- `zoom` - View specific screen regions at full resolution (Claude Opus 4.5 only) + +```typescript +import { ChatAnthropic, tools } from "@langchain/anthropic"; + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +const computer = tools.computer_20250124({ + // Required: specify display dimensions + displayWidthPx: 1024, + displayHeightPx: 768, + // Optional: X11 display number + displayNumber: 1, + execute: async (action) => { + switch (action.action) { + case "screenshot": + // Capture and return base64-encoded screenshot + // ... + case "left_click": + // Click at the specified coordinates + // ... + // ... + } + }, +}); + +const llmWithComputer = llm.bindTools([computer]); + +const response = await llmWithComputer.invoke( + "Save a picture of a cat to my desktop." +); +``` + +For Claude Opus 4.5 with zoom support: + +```typescript +import { tools } from "@langchain/anthropic"; + +const computer = tools.computer_20251124({ + displayWidthPx: 1920, + displayHeightPx: 1080, + // Enable zoom for detailed screen region inspection + enableZoom: true, + execute: async (action) => { + // Handle actions including "zoom" for Claude Opus 4.5 + // ... + }, +}); +``` + +For more information, see [Anthropic's Computer Use documentation](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/computer-use). + +### Code Execution Tool + +The code execution tool (`codeExecution_20250825`) allows Claude to run Bash commands and manipulate files in a secure, sandboxed environment. Claude can analyze data, create visualizations, perform calculations, and process files. + +When this tool is provided, Claude automatically gains access to: + +- **Bash commands** - Execute shell commands for system operations +- **File operations** - Create, view, and edit files directly + +```typescript +import { ChatAnthropic, tools } from "@langchain/anthropic"; + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +// Basic usage - calculations and data analysis +const response = await llm.invoke( + "Calculate the mean and standard deviation of [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]", + { tools: [tools.codeExecution_20250825()] } +); + +// File operations and visualization +const response2 = await llm.invoke( + "Create a matplotlib visualization of sales data and save it as chart.png", + { tools: [tools.codeExecution_20250825()] } +); +``` + +Container reuse for multi-step workflows: + +```typescript +// First request - creates a container +const response1 = await llm.invoke("Write a random number to /tmp/number.txt", { + tools: [tools.codeExecution_20250825()], +}); + +// Extract container ID from response for reuse +const containerId = response1.response_metadata?.container?.id; + +// Second request - reuse container to access the file +const response2 = await llm.invoke( + "Read /tmp/number.txt and calculate its square", + { + tools: [tools.codeExecution_20250825()], + container: containerId, + } +); +``` + +For more information, see [Anthropic's Code Execution Tool documentation](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/code-execution-tool). + +### Bash Tool + +The bash tool (`bash_20250124`) enables shell command execution in a persistent bash session. Unlike the sandboxed code execution tool, this tool requires you to provide your own execution environment. + +> **⚠️ Security Warning:** The bash tool provides direct system access. Implement safety measures such as running in isolated environments (Docker/VM), command filtering, and resource limits. + +The bash tool provides: + +- **Persistent bash session** - Maintains state between commands +- **Shell command execution** - Run any shell command +- **Environment access** - Access to environment variables and working directory +- **Command chaining** - Support for pipes, redirects, and scripting + +Available commands: + +- Execute a command: `{ command: "ls -la" }` +- Restart the session: `{ restart: true }` + +```typescript +import { ChatAnthropic, tools } from "@langchain/anthropic"; +import { execSync } from "child_process"; + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +const bash = tools.bash_20250124({ + execute: async (args) => { + if (args.restart) { + // Reset session state + return "Bash session restarted"; + } + try { + const output = execSync(args.command, { + encoding: "utf-8", + timeout: 30000, + }); + return output; + } catch (error) { + return `Error: ${(error as Error).message}`; + } + }, +}); + +const llmWithBash = llm.bindTools([bash]); + +const response = await llmWithBash.invoke( + "List all Python files in the current directory" +); + +// Process tool calls and execute commands +console.log(response.tool_calls?.[0].name); // "bash" +console.log(response.tool_calls?.[0].args.command); // "ls -la *.py" +``` + +For more information, see [Anthropic's Bash Tool documentation](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/bash-tool). + +### MCP Toolset + +The MCP toolset (`mcpToolset_20251120`) enables Claude to connect to remote MCP (Model Context Protocol) servers directly from the Messages API without implementing a separate MCP client. This allows Claude to use tools provided by MCP servers. + +Key features: + +- **Direct API integration** - Connect to MCP servers without implementing an MCP client +- **Tool calling support** - Access MCP tools through the Messages API +- **Flexible tool configuration** - Enable all tools, allowlist specific tools, or denylist unwanted tools +- **Per-tool configuration** - Configure individual tools with custom settings +- **OAuth authentication** - Support for OAuth Bearer tokens for authenticated servers +- **Multiple servers** - Connect to multiple MCP servers in a single request + +```typescript +import { ChatAnthropic, tools } from "@langchain/anthropic"; + +const llm = new ChatAnthropic({ + model: "claude-sonnet-4-5-20250929", +}); + +// Basic usage - enable all tools from an MCP server +const response = await llm.invoke("What tools do you have available?", { + mcp_servers: [ + { + type: "url", + url: "https://example-server.modelcontextprotocol.io/sse", + name: "example-mcp", + authorization_token: "YOUR_TOKEN", + }, + ], + tools: [tools.mcpToolset_20251120({ serverName: "example-mcp" })], +}); +``` + +**Allowlist pattern** - Enable only specific tools: + +```typescript +const response = await llm.invoke("Search for events", { + mcp_servers: [ + { + type: "url", + url: "https://calendar.example.com/sse", + name: "google-calendar-mcp", + authorization_token: "YOUR_TOKEN", + }, + ], + tools: [ + tools.mcpToolset_20251120({ + serverName: "google-calendar-mcp", + // Disable all tools by default + defaultConfig: { enabled: false }, + // Explicitly enable only these tools + configs: { + search_events: { enabled: true }, + create_event: { enabled: true }, + }, + }), + ], +}); +``` + +**Denylist pattern** - Disable specific tools: + +```typescript +const response = await llm.invoke("List my events", { + mcp_servers: [ + { + type: "url", + url: "https://calendar.example.com/sse", + name: "google-calendar-mcp", + authorization_token: "YOUR_TOKEN", + }, + ], + tools: [ + tools.mcpToolset_20251120({ + serverName: "google-calendar-mcp", + // All tools enabled by default, just disable dangerous ones + configs: { + delete_all_events: { enabled: false }, + share_calendar_publicly: { enabled: false }, + }, + }), + ], +}); +``` + +**Multiple MCP servers**: + +```typescript +const response = await llm.invoke("Use tools from both servers", { + mcp_servers: [ + { + type: "url", + url: "https://mcp.example1.com/sse", + name: "mcp-server-1", + authorization_token: "TOKEN1", + }, + { + type: "url", + url: "https://mcp.example2.com/sse", + name: "mcp-server-2", + authorization_token: "TOKEN2", + }, + ], + tools: [ + tools.mcpToolset_20251120({ serverName: "mcp-server-1" }), + tools.mcpToolset_20251120({ + serverName: "mcp-server-2", + defaultConfig: { deferLoading: true }, + }), + ], +}); +``` + +**With Tool Search** - Use deferred loading for on-demand tool discovery: + +```typescript +const response = await llm.invoke("Find and use the right tool", { + mcp_servers: [ + { + type: "url", + url: "https://example.com/sse", + name: "example-mcp", + }, + ], + tools: [ + tools.toolSearchRegex_20251119(), + tools.mcpToolset_20251120({ + serverName: "example-mcp", + defaultConfig: { deferLoading: true }, + }), + ], +}); +``` + +For more information, see [Anthropic's MCP Connector documentation](https://docs.anthropic.com/en/docs/agents-and-tools/mcp-connector). diff --git a/src/oss/javascript/integrations/tools/openai.mdx b/src/oss/javascript/integrations/tools/openai.mdx new file mode 100644 index 0000000000..f4dff84378 --- /dev/null +++ b/src/oss/javascript/integrations/tools/openai.mdx @@ -0,0 +1,584 @@ +--- +title: Tools +--- + +The `@langchain/openai` package provides LangChain-compatible wrappers for OpenAI's built-in tools. These tools can be bound to `ChatOpenAI` using `bindTools()` or @[`createAgent`]. + +### Web Search Tool + +The web search tool allows OpenAI models to search the web for up-to-date information before generating a response. Web search supports three main types: + +1. **Non-reasoning web search**: Quick lookups where the model passes queries directly to the search tool +2. **Agentic search with reasoning models**: The model actively manages the search process, analyzing results and deciding whether to keep searching +3. **Deep research**: Extended investigations using models like `o3-deep-research` or `gpt-5` with high reasoning effort + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; + +const model = new ChatOpenAI({ + model: "gpt-4o", +}); + +// Basic usage +const response = await model.invoke( + "What was a positive news story from today?", + { + tools: [tools.webSearch()], + } +); +``` + +**Domain filtering** - Limit search results to specific domains (up to 100): + +```typescript +const response = await model.invoke("Latest AI research news", { + tools: [ + tools.webSearch({ + filters: { + allowedDomains: ["arxiv.org", "nature.com", "science.org"], + }, + }), + ], +}); +``` + +**User location** - Refine search results based on geography: + +```typescript +const response = await model.invoke("What are the best restaurants near me?", { + tools: [ + tools.webSearch({ + userLocation: { + type: "approximate", + country: "US", + city: "San Francisco", + region: "California", + timezone: "America/Los_Angeles", + }, + }), + ], +}); +``` + +**Cache-only mode** - Disable live internet access: + +```typescript +const response = await model.invoke("Find information about OpenAI", { + tools: [ + tools.webSearch({ + externalWebAccess: false, + }), + ], +}); +``` + +For more information, see [OpenAI's Web Search Documentation](https://platform.openai.com/docs/guides/tools-web-search). + +### MCP Tool (Model Context Protocol) + +The MCP tool allows OpenAI models to connect to remote MCP servers and OpenAI-maintained service connectors, giving models access to external tools and services. + +There are two ways to use MCP tools: + +1. **Remote MCP servers**: Connect to any public MCP server via URL +2. **Connectors**: Use OpenAI-maintained wrappers for popular services like Google Workspace or Dropbox + +**Remote MCP server** - Connect to any MCP-compatible server: + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; + +const model = new ChatOpenAI({ model: "gpt-4o" }); + +const response = await model.invoke("Roll 2d4+1", { + tools: [ + tools.mcp({ + serverLabel: "dmcp", + serverDescription: "A D&D MCP server for dice rolling", + serverUrl: "https://dmcp-server.deno.dev/sse", + requireApproval: "never", + }), + ], +}); +``` + +**Service connectors** - Use OpenAI-maintained connectors for popular services: + +```typescript +const response = await model.invoke("What's on my calendar today?", { + tools: [ + tools.mcp({ + serverLabel: "google_calendar", + connectorId: "connector_googlecalendar", + authorization: "", + requireApproval: "never", + }), + ], +}); +``` + +For more information, see [OpenAI's MCP Documentation](https://platform.openai.com/docs/guides/tools-remote-mcp). + +### Code Interpreter Tool + +The Code Interpreter tool allows models to write and run Python code in a sandboxed environment to solve complex problems. + +Use Code Interpreter for: + +- **Data analysis**: Processing files with diverse data and formatting +- **File generation**: Creating files with data and images of graphs +- **Iterative coding**: Writing and running code iteratively to solve problems +- **Visual intelligence**: Cropping, zooming, rotating, and transforming images + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; + +const model = new ChatOpenAI({ model: "gpt-4.1" }); + +// Basic usage with auto container (default 1GB memory) +const response = await model.invoke("Solve the equation 3x + 11 = 14", { + tools: [tools.codeInterpreter()], +}); +``` + +**Memory configuration** - Choose from 1GB (default), 4GB, 16GB, or 64GB: + +```typescript +const response = await model.invoke( + "Analyze this large dataset and create visualizations", + { + tools: [ + tools.codeInterpreter({ + container: { memoryLimit: "4g" }, + }), + ], + } +); +``` + +**With files** - Make uploaded files available to the code: + +```typescript +const response = await model.invoke("Process the uploaded CSV file", { + tools: [ + tools.codeInterpreter({ + container: { + memoryLimit: "4g", + fileIds: ["file-abc123", "file-def456"], + }, + }), + ], +}); +``` + +**Explicit container** - Use a pre-created container ID: + +```typescript +const response = await model.invoke("Continue working with the data", { + tools: [ + tools.codeInterpreter({ + container: "cntr_abc123", + }), + ], +}); +``` + +> **Note**: Containers expire after 20 minutes of inactivity. While called "Code Interpreter", the model knows it as the "python tool" - for explicit prompting, ask for "the python tool" in your prompts. + +For more information, see [OpenAI's Code Interpreter Documentation](https://platform.openai.com/docs/guides/tools-code-interpreter). + +### File Search Tool + +The File Search tool allows models to search your files for relevant information using semantic and keyword search. It enables retrieval from a knowledge base of previously uploaded files stored in vector stores. + +**Prerequisites**: Before using File Search, you must: + +1. Upload files to the File API with `purpose: "assistants"` +2. Create a vector store +3. Add files to the vector store + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; + +const model = new ChatOpenAI({ model: "gpt-4.1" }); + +const response = await model.invoke("What is deep research by OpenAI?", { + tools: [ + tools.fileSearch({ + vectorStoreIds: ["vs_abc123"], + // maxNumResults: 5, // Limit results for lower latency + // filters: { type: "eq", key: "category", value: "blog" }, // Metadata filtering + // filters: { type: "and", filters: [ // Compound filters (AND/OR) + // { type: "eq", key: "category", value: "technical" }, + // { type: "gte", key: "year", value: 2024 }, + // ]}, + // rankingOptions: { scoreThreshold: 0.8, ranker: "auto" }, // Customize scoring + }), + ], +}); +``` + +Filter operators: `eq` (equals), `ne` (not equal), `gt` (greater than), `gte` (greater than or equal), `lt` (less than), `lte` (less than or equal). + +For more information, see [OpenAI's File Search Documentation](https://platform.openai.com/docs/guides/tools-file-search). + +### Image Generation Tool + +The Image Generation tool allows models to generate or edit images using text prompts and optional image inputs. It leverages the GPT Image model and automatically optimizes text inputs for improved performance. + +Use Image Generation for: + +- **Creating images from text**: Generate images from detailed text descriptions +- **Editing existing images**: Modify images based on text instructions +- **Multi-turn image editing**: Iteratively refine images across conversation turns +- **Various output formats**: Support for PNG, JPEG, and WebP formats + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; + +const model = new ChatOpenAI({ model: "gpt-4o" }); + +// Basic usage - generate an image +const response = await model.invoke( + "Generate an image of a gray tabby cat hugging an otter with an orange scarf", + { tools: [tools.imageGeneration()] } +); + +// Access the generated image (base64-encoded) +const imageOutput = response.additional_kwargs.tool_outputs?.find( + (output) => output.type === "image_generation_call" +); +if (imageOutput?.result) { + const fs = await import("fs"); + fs.writeFileSync("output.png", Buffer.from(imageOutput.result, "base64")); +} +``` + +**Custom size and quality** - Configure output dimensions and quality: + +```typescript +const response = await model.invoke("Draw a beautiful sunset over mountains", { + tools: [ + tools.imageGeneration({ + size: "1536x1024", // Landscape format (also: "1024x1024", "1024x1536", "auto") + quality: "high", // Quality level (also: "low", "medium", "auto") + }), + ], +}); +``` + +**Output format and compression** - Choose format and compression level: + +```typescript +const response = await model.invoke("Create a product photo", { + tools: [ + tools.imageGeneration({ + outputFormat: "jpeg", // Format (also: "png", "webp") + outputCompression: 90, // Compression 0-100 (for JPEG/WebP) + }), + ], +}); +``` + +**Transparent background** - Generate images with transparency: + +```typescript +const response = await model.invoke( + "Create a logo with transparent background", + { + tools: [ + tools.imageGeneration({ + background: "transparent", // Background type (also: "opaque", "auto") + outputFormat: "png", + }), + ], + } +); +``` + +**Streaming with partial images** - Get visual feedback during generation: + +```typescript +const response = await model.invoke("Draw a detailed fantasy castle", { + tools: [ + tools.imageGeneration({ + partialImages: 2, // Number of partial images (0-3) + }), + ], +}); +``` + +**Force image generation** - Ensure the model uses the image generation tool: + +```typescript +const response = await model.invoke("A serene lake at dawn", { + tools: [tools.imageGeneration()], + tool_choice: { type: "image_generation" }, +}); +``` + +**Multi-turn editing** - Refine images across conversation turns: + +```typescript +// First turn: generate initial image +const response1 = await model.invoke("Draw a red car", { + tools: [tools.imageGeneration()], +}); + +// Second turn: edit the image +const response2 = await model.invoke( + [response1, new HumanMessage("Now change the car color to blue")], + { tools: [tools.imageGeneration()] } +); +``` + +> **Prompting tips**: Use terms like "draw" or "edit" for best results. For combining images, say "edit the first image by adding this element" instead of "combine" or "merge". + +Supported models: `gpt-4o`, `gpt-4o-mini`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-nano`, `o3` + +For more information, see [OpenAI's Image Generation Documentation](https://platform.openai.com/docs/guides/tools-image-generation). + +### Computer Use Tool + +The Computer Use tool allows models to control computer interfaces by simulating mouse clicks, keyboard input, scrolling, and more. It uses OpenAI's Computer-Using Agent (CUA) model to understand screenshots and suggest actions. + +> **Beta**: Computer use is in beta. Use in sandboxed environments only and do not use for high-stakes or authenticated tasks. Always implement human-in-the-loop for important decisions. + +**How it works**: The tool operates in a continuous loop: + +1. Model sends computer actions (click, type, scroll, etc.) +2. Your code executes these actions in a controlled environment +3. You capture a screenshot of the result +4. Send the screenshot back to the model +5. Repeat until the task is complete + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; + +const model = new ChatOpenAI({ model: "computer-use-preview" }); + +// With execute callback for automatic action handling +const computer = tools.computerUse({ + displayWidth: 1024, + displayHeight: 768, + environment: "browser", + execute: async (action) => { + if (action.type === "screenshot") { + return captureScreenshot(); + } + if (action.type === "click") { + await page.mouse.click(action.x, action.y, { button: action.button }); + return captureScreenshot(); + } + if (action.type === "type") { + await page.keyboard.type(action.text); + return captureScreenshot(); + } + if (action.type === "scroll") { + await page.mouse.move(action.x, action.y); + await page.evaluate( + `window.scrollBy(${action.scroll_x}, ${action.scroll_y})` + ); + return captureScreenshot(); + } + // Handle other actions... + return captureScreenshot(); + }, +}); + +const llmWithComputer = model.bindTools([computer]); +const response = await llmWithComputer.invoke( + "Check the latest news on bing.com" +); +``` + +For more information, see [OpenAI's Computer Use Documentation](https://platform.openai.com/docs/guides/tools-computer-use). + +### Local Shell Tool + +The Local Shell tool allows models to run shell commands locally on a machine you provide. Commands are executed inside your own runtime—the API only returns the instructions. + +> **Security Warning**: Running arbitrary shell commands can be dangerous. Always sandbox execution or add strict allow/deny-lists before forwarding commands to the system shell. +> **Note**: This tool is designed to work with [Codex CLI](https://github.com/openai/codex) and the `codex-mini-latest` model. + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; +import { exec } from "child_process"; +import { promisify } from "util"; + +const execAsync = promisify(exec); +const model = new ChatOpenAI({ model: "codex-mini-latest" }); + +// With execute callback for automatic command handling +const shell = tools.localShell({ + execute: async (action) => { + const { command, env, working_directory, timeout_ms } = action; + const result = await execAsync(command.join(" "), { + cwd: working_directory ?? process.cwd(), + env: { ...process.env, ...env }, + timeout: timeout_ms ?? undefined, + }); + return result.stdout + result.stderr; + }, +}); + +const llmWithShell = model.bindTools([shell]); +const response = await llmWithShell.invoke( + "List files in the current directory" +); +``` + +**Action properties**: The model returns actions with these properties: + +- `command` - Array of argv tokens to execute +- `env` - Environment variables to set +- `working_directory` - Directory to run the command in +- `timeout_ms` - Suggested timeout (enforce your own limits) +- `user` - Optional user to run the command as + +For more information, see [OpenAI's Local Shell Documentation](https://platform.openai.com/docs/guides/tools-local-shell). + +### Shell Tool + +The Shell tool allows models to run shell commands through your integration. Unlike Local Shell, this tool supports executing multiple commands concurrently and is designed for `gpt-5.1`. + +> **Security Warning**: Running arbitrary shell commands can be dangerous. Always sandbox execution or add strict allow/deny-lists before forwarding commands to the system shell. + +**Use cases**: + +- **Automating filesystem or process diagnostics** – e.g., "find the largest PDF under ~/Documents" +- **Extending model capabilities** – Using built-in UNIX utilities, Python runtime, and other CLIs +- **Running multi-step build and test flows** – Chaining commands like `pip install` and `pytest` +- **Complex agentic coding workflows** – Using with `apply_patch` for file operations + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; +import { exec } from "node:child_process/promises"; + +const model = new ChatOpenAI({ model: "gpt-5.1" }); + +// With execute callback for automatic command handling +const shellTool = tools.shell({ + execute: async (action) => { + const outputs = await Promise.all( + action.commands.map(async (cmd) => { + try { + const { stdout, stderr } = await exec(cmd, { + timeout: action.timeout_ms ?? undefined, + }); + return { + stdout, + stderr, + outcome: { type: "exit" as const, exit_code: 0 }, + }; + } catch (error) { + const timedOut = error.killed && error.signal === "SIGTERM"; + return { + stdout: error.stdout ?? "", + stderr: error.stderr ?? String(error), + outcome: timedOut + ? { type: "timeout" as const } + : { type: "exit" as const, exit_code: error.code ?? 1 }, + }; + } + }) + ); + return { + output: outputs, + maxOutputLength: action.max_output_length, + }; + }, +}); + +const llmWithShell = model.bindTools([shellTool]); +const response = await llmWithShell.invoke( + "Find the largest PDF file in ~/Documents" +); +``` + +**Action properties**: The model returns actions with these properties: + +- `commands` - Array of shell commands to execute (can run concurrently) +- `timeout_ms` - Optional timeout in milliseconds (enforce your own limits) +- `max_output_length` - Optional maximum characters to return per command + +**Return format**: Your execute function should return a `ShellResult`: + +```typescript +interface ShellResult { + output: Array<{ + stdout: string; + stderr: string; + outcome: { type: "exit"; exit_code: number } | { type: "timeout" }; + }>; + maxOutputLength?: number | null; // Pass back from action if provided +} +``` + +> **Note**: Only available through the Responses API with `gpt-5.1`. The `timeout_ms` from the model is only a hint—always enforce your own limits. + +For more information, see [OpenAI's Shell Documentation](https://platform.openai.com/docs/guides/tools-shell). + +### Apply Patch Tool + +The Apply Patch tool allows models to propose structured diffs that your integration applies. This enables iterative, multi-step code editing workflows where the model can create, update, and delete files in your codebase. + +**When to use**: + +- **Multi-file refactors** – Rename symbols, extract helpers, or reorganize modules +- **Bug fixes** – Have the model both diagnose issues and emit precise patches +- **Tests & docs generation** – Create new test files, fixtures, and documentation +- **Migrations & mechanical edits** – Apply repetitive, structured updates + +> **Security Warning**: Applying patches can modify files in your codebase. Always validate paths, implement backups, and consider sandboxing. +> **Note**: This tool is designed to work with `gpt-5.1` model. + +```typescript +import { ChatOpenAI, tools } from "@langchain/openai"; +import { applyDiff } from "@openai/agents"; +import * as fs from "fs/promises"; + +const model = new ChatOpenAI({ model: "gpt-5.1" }); + +// With execute callback for automatic patch handling +const patchTool = tools.applyPatch({ + execute: async (operation) => { + if (operation.type === "create_file") { + const content = applyDiff("", operation.diff, "create"); + await fs.writeFile(operation.path, content); + return `Created ${operation.path}`; + } + if (operation.type === "update_file") { + const current = await fs.readFile(operation.path, "utf-8"); + const newContent = applyDiff(current, operation.diff); + await fs.writeFile(operation.path, newContent); + return `Updated ${operation.path}`; + } + if (operation.type === "delete_file") { + await fs.unlink(operation.path); + return `Deleted ${operation.path}`; + } + return "Unknown operation type"; + }, +}); + +const llmWithPatch = model.bindTools([patchTool]); +const response = await llmWithPatch.invoke( + "Rename the fib() function to fibonacci() in lib/fib.py" +); +``` + +**Operation types**: The model returns operations with these properties: + +- `create_file` – Create a new file at `path` with content from `diff` +- `update_file` – Modify an existing file at `path` using V4A diff format in `diff` +- `delete_file` – Remove a file at `path` + +**Best practices**: + +- **Path validation**: Prevent directory traversal and restrict edits to allowed directories +- **Backups**: Consider backing up files before applying patches +- **Error handling**: Return descriptive error messages so the model can recover +- **Atomicity**: Decide whether you want "all-or-nothing" semantics (rollback if any patch fails) + +For more information, see [OpenAI's Apply Patch Documentation](https://platform.openai.com/docs/guides/tools-apply-patch).