feat: provider-independent web search for all LLM backends

OpenAI: uses native web_search_options (unchanged)
Other providers: DuckDuckGo HTML search (no API key) → fetch top results → inject into prompt

Auditor eraResearch now works with Anthropic, GLM, Qwen, Kimi, etc.

packages/core/src/agents/base.ts

@@ -1,5 +1,6 @@
 import type { LLMClient, LLMMessage, LLMResponse, OnStreamProgress } from "../llm/provider.js";
 import { chatCompletion } from "../llm/provider.js";
+import { searchWeb, fetchUrl } from "../utils/web-search.js";
 import type { Logger } from "../utils/logger.js";
 
 export interface AgentContext {
@@ -18,6 +19,10 @@ export abstract class BaseAgent {
     this.ctx = ctx;
   }
 
+  protected get log() {
+    return this.ctx.logger;
+  }
+
   protected async chat(
     messages: ReadonlyArray<LLMMessage>,
     options?: { readonly temperature?: number; readonly maxTokens?: number },
@@ -29,20 +34,66 @@ export abstract class BaseAgent {
   }
 
   /**
-   * Chat with provider-native web search enabled.
-   * OpenAI Chat API: uses web_search_options.
-   * OpenAI Responses API: uses web_search_preview hosted tool.
-   * Anthropic: falls back to regular chat (no native search).
+   * Chat with web search enabled.
+   * OpenAI: uses native web_search_options / web_search_preview.
+   * Other providers: searches via DuckDuckGo, fetches top results, injects into prompt.
    */
   protected async chatWithSearch(
     messages: ReadonlyArray<LLMMessage>,
     options?: { readonly temperature?: number; readonly maxTokens?: number },
   ): Promise<LLMResponse> {
-    return chatCompletion(this.ctx.client, this.ctx.model, messages, {
-      ...options,
-      webSearch: true,
-      onStreamProgress: this.ctx.onStreamProgress,
-    });
+    // OpenAI has native search — use it directly
+    if (this.ctx.client.provider === "openai") {
+      return chatCompletion(this.ctx.client, this.ctx.model, messages, {
+        ...options,
+        webSearch: true,
+        onStreamProgress: this.ctx.onStreamProgress,
+      });
+    }
+
+    // Other providers: self-hosted search → inject results into prompt
+    const lastUserMsg = [...messages].reverse().find((m) => m.role === "user");
+    if (!lastUserMsg) {
+      return this.chat(messages, options);
+    }
+
+    try {
+      // Extract search query from user message (first 200 chars)
+      const query = lastUserMsg.content.slice(0, 200);
+      this.log?.info(`[search] Searching: ${query.slice(0, 60)}...`);
+
+      const results = await searchWeb(query, 3);
+      if (results.length === 0) {
+        this.log?.warn("[search] No results found, falling back to regular chat");
+        return this.chat(messages, options);
+      }
+
+      // Fetch top result for full content
+      let fullContent = "";
+      try {
+        fullContent = await fetchUrl(results[0]!.url, 4000);
+      } catch {
+        // Fetch failed, use snippets only
+      }
+
+      const searchContext = [
+        "## Web Search Results\n",
+        ...results.map((r, i) => `${i + 1}. **${r.title}**\n   ${r.url}\n   ${r.snippet}`),
+        ...(fullContent ? [`\n## Full Content (Top Result)\n${fullContent}`] : []),
+      ].join("\n");
+
+      // Inject search results before the last user message
+      const augmentedMessages: LLMMessage[] = messages.map((m) =>
+        m === lastUserMsg
+          ? { ...m, content: `${searchContext}\n\n---\n\n${m.content}` }
+          : m,
+      );
+
+      return this.chat(augmentedMessages, options);
+    } catch (e) {
+      this.log?.warn(`[search] Search failed: ${e}, falling back to regular chat`);
+      return this.chat(messages, options);
+    }
   }
 
   abstract get name(): string;

packages/core/src/index.ts

@@ -36,7 +36,7 @@ export { getFanficDimensionConfig, FANFIC_DIMENSIONS, type FanficDimensionConfig
 export { buildFanficCanonSection, buildCharacterVoiceProfiles, buildFanficModeInstructions } from "./agents/fanfic-prompt-sections.js";
 
 // Utils
-export { fetchUrl } from "./utils/web-search.js";
+export { fetchUrl, searchWeb } from "./utils/web-search.js";
 export { splitChapters, type SplitChapter } from "./utils/chapter-splitter.js";
 export { createLogger, createStderrSink, createJsonLineSink, nullSink, type Logger, type LogSink, type LogLevel, type LogEntry } from "./utils/logger.js";
 export { loadProjectConfig, GLOBAL_CONFIG_DIR, GLOBAL_ENV_PATH } from "./utils/config-loader.js";

packages/core/src/utils/web-search.ts

@@ -1,11 +1,71 @@
 /**
- * URL fetch utility for agent loop.
+ * Web search + URL fetch utilities.
  *
- * Web search is handled natively by the LLM provider (OpenAI web_search_preview
- * or web_search_options). This module only provides URL fetching for cases where
- * the agent loop needs to read a specific page.
+ * searchWeb(): Provider-independent web search via DuckDuckGo HTML (no API key needed).
+ * fetchUrl(): Fetch a specific URL and return plain text.
  */
 
+export interface SearchResult {
+  readonly title: string;
+  readonly url: string;
+  readonly snippet: string;
+}
+
+/**
+ * Search the web via DuckDuckGo HTML. No API key required.
+ * Returns up to `maxResults` results with title, url, snippet.
+ */
+export async function searchWeb(query: string, maxResults = 5): Promise<ReadonlyArray<SearchResult>> {
+  const encodedQuery = encodeURIComponent(query);
+  const res = await fetch(`https://html.duckduckgo.com/html/?q=${encodedQuery}`, {
+    headers: {
+      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+      "Accept": "text/html",
+    },
+    signal: AbortSignal.timeout(10000),
+  });
+
+  if (!res.ok) {
+    throw new Error(`DuckDuckGo search failed: ${res.status}`);
+  }
+
+  const html = await res.text();
+  const results: SearchResult[] = [];
+
+  // Parse DuckDuckGo HTML results — each result is in a <div class="result">
+  const resultBlocks = html.split(/class="result(?:\s|")/);
+  for (let i = 1; i < resultBlocks.length && results.length < maxResults; i++) {
+    const block = resultBlocks[i]!;
+
+    // Extract URL from <a class="result__a" href="...">
+    const urlMatch = block.match(/href="([^"]+)"[^>]*class="result__a"|class="result__a"[^>]*href="([^"]+)"/);
+    const rawUrl = urlMatch?.[1] ?? urlMatch?.[2] ?? "";
+
+    // DuckDuckGo wraps URLs in redirects — extract actual URL
+    const actualUrlMatch = rawUrl.match(/uddg=([^&]+)/);
+    const url = actualUrlMatch ? decodeURIComponent(actualUrlMatch[1]!) : rawUrl;
+
+    if (!url || url.startsWith("/") || !url.startsWith("http")) continue;
+
+    // Extract title
+    const titleMatch = block.match(/class="result__a"[^>]*>([^<]+)/);
+    const title = titleMatch?.[1]?.trim() ?? "";
+
+    // Extract snippet from <a class="result__snippet">
+    const snippetMatch = block.match(/class="result__snippet"[^>]*>([\s\S]*?)(?:<\/a>|<\/td>)/);
+    const snippet = snippetMatch?.[1]
+      ?.replace(/<[^>]*>/g, "")
+      ?.replace(/\s+/g, " ")
+      ?.trim() ?? "";
+
+    if (title && url) {
+      results.push({ title, url, snippet });
+    }
+  }
+
+  return results;
+}
+
 /**
  * Fetch a URL and return its text content.
  * HTML is stripped to plain text. Output is truncated to maxChars.