refactor: remove DuckDuckGo, use Tavily as sole search provider

packages/core/src/agents/base.ts

@@ -36,7 +36,7 @@ export abstract class BaseAgent {
   /**
    * Chat with web search enabled.
    * OpenAI: uses native web_search_options / web_search_preview.
-   * Other providers: searches via DuckDuckGo, fetches top results, injects into prompt.
+   * Other providers: searches via Tavily API (TAVILY_API_KEY), injects results into prompt.
    */
   protected async chatWithSearch(
     messages: ReadonlyArray<LLMMessage>,

packages/core/src/utils/web-search.ts

@@ -1,7 +1,7 @@
 /**
  * Web search + URL fetch utilities.
  *
- * searchWeb(): Provider-independent web search via DuckDuckGo HTML (no API key needed).
+ * searchWeb(): Tavily API search (requires TAVILY_API_KEY env var).
  * fetchUrl(): Fetch a specific URL and return plain text.
  */
 
@@ -12,19 +12,16 @@ export interface SearchResult {
 }
 
 /**
- * Search the web. Priority:
- * 1. Tavily API (if TAVILY_API_KEY is set) — clean, structured, AI-optimized
- * 2. DuckDuckGo HTML fallback — free, no key needed
+ * Search the web via Tavily API.
+ * Requires TAVILY_API_KEY environment variable.
+ * Throws if key is not set — caller should catch and fall back to regular chat.
  */
 export async function searchWeb(query: string, maxResults = 5): Promise<ReadonlyArray<SearchResult>> {
-  const tavilyKey = process.env.TAVILY_API_KEY;
-  if (tavilyKey) {
-    return searchViaTavily(query, tavilyKey, maxResults);
+  const apiKey = process.env.TAVILY_API_KEY;
+  if (!apiKey) {
+    throw new Error("TAVILY_API_KEY not set. Set this env var to enable web search, or use OpenAI which has native search.");
   }
-  return searchViaDuckDuckGo(query, maxResults);
-}
 
-async function searchViaTavily(query: string, apiKey: string, maxResults: number): Promise<SearchResult[]> {
   const res = await fetch("https://api.tavily.com/search", {
     method: "POST",
     headers: {
@@ -51,51 +48,6 @@ async function searchViaTavily(query: string, apiKey: string, maxResults: number
   }));
 }
 
-async function searchViaDuckDuckGo(query: string, maxResults: number): Promise<SearchResult[]> {
-  const encodedQuery = encodeURIComponent(query);
-  const res = await fetch(`https://html.duckduckgo.com/html/?q=${encodedQuery}`, {
-    headers: {
-      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
-      "Accept": "text/html",
-    },
-    signal: AbortSignal.timeout(10000),
-  });
-
-  if (!res.ok) {
-    throw new Error(`DuckDuckGo search failed: ${res.status}`);
-  }
-
-  const html = await res.text();
-  const results: SearchResult[] = [];
-
-  const resultBlocks = html.split(/class="result(?:\s|")/);
-  for (let i = 1; i < resultBlocks.length && results.length < maxResults; i++) {
-    const block = resultBlocks[i]!;
-
-    const urlMatch = block.match(/href="([^"]+)"[^>]*class="result__a"|class="result__a"[^>]*href="([^"]+)"/);
-    const rawUrl = urlMatch?.[1] ?? urlMatch?.[2] ?? "";
-    const actualUrlMatch = rawUrl.match(/uddg=([^&]+)/);
-    const url = actualUrlMatch ? decodeURIComponent(actualUrlMatch[1]!) : rawUrl;
-
-    if (!url || url.startsWith("/") || !url.startsWith("http")) continue;
-
-    const titleMatch = block.match(/class="result__a"[^>]*>([^<]+)/);
-    const title = titleMatch?.[1]?.trim() ?? "";
-
-    const snippetMatch = block.match(/class="result__snippet"[^>]*>([\s\S]*?)(?:<\/a>|<\/td>)/);
-    const snippet = snippetMatch?.[1]
-      ?.replace(/<[^>]*>/g, "")
-      ?.replace(/\s+/g, " ")
-      ?.trim() ?? "";
-
-    if (title && url) {
-      results.push({ title, url, snippet });
-    }
-  }
-
-  return results;
-}
-
 /**
  * Fetch a URL and return its text content.
  * HTML is stripped to plain text. Output is truncated to maxChars.
@@ -116,7 +68,6 @@ export async function fetchUrl(url: string, maxChars = 8000): Promise<string> {
   const contentType = res.headers.get("content-type") ?? "";
   const text = await res.text();
 
-  // If HTML, strip tags and collapse whitespace
   if (contentType.includes("html")) {
     return text
       .replace(/<script[\s\S]*?<\/script>/gi, "")