feat(server): replace existing search functionality with searxng library

package.json

@@ -41,6 +41,7 @@
     "html-to-text": "^9.0.5",
     "langchain": "^0.1.30",
     "pdf-parse": "^1.1.1",
+    "searxng": "^0.0.5",
     "winston": "^3.13.0",
     "ws": "^8.17.1",
     "zod": "^3.22.4"

src/agents/academicSearchAgent.ts

@@ -113,11 +113,11 @@ const createBasicAcademicSearchRetrieverChain = (llm: BaseChatModel) => {
       }
 
       const res = await searchSearxng(input, {
-        language: 'en',
+        lang: 'en',
         engines: [
           'arxiv',
-          'google scholar',
-          'internetarchivescholar',
+          'google_scholar',
+          'internet_archive_scholar',
           'pubmed',
         ],
       });

src/agents/imageSearchAgent.ts

@@ -53,7 +53,7 @@ const createImageSearchChain = (llm: BaseChatModel) => {
     strParser,
     RunnableLambda.from(async (input: string) => {
       const res = await searchSearxng(input, {
-        engines: ['bing images', 'google images'],
+        engines: ['bing_images', 'google_images'],
       });
 
       const images = [];

src/agents/redditSearchAgent.ts

@@ -113,7 +113,7 @@ const createBasicRedditSearchRetrieverChain = (llm: BaseChatModel) => {
       }
 
       const res = await searchSearxng(input, {
-        language: 'en',
+        lang: 'en',
         engines: ['reddit'],
       });
 

src/agents/videoSearchAgent.ts

@@ -53,7 +53,7 @@ const createVideoSearchChain = (llm: BaseChatModel) => {
     strParser,
     RunnableLambda.from(async (input: string) => {
       const res = await searchSearxng(input, {
-        engines: ['youtube'],
+        engines: ['youtube_api'],
       });
 
       const videos = [];

src/agents/webSearchAgent.ts

@@ -218,7 +218,7 @@ const createBasicWebSearchRetrieverChain = (llm: BaseChatModel) => {
         return { query: question, docs: docs };
       } else {
         const res = await searchSearxng(input, {
-          language: 'en',
+          lang: 'en',
         });
 
         const documents = res.results.map(

src/agents/wolframAlphaSearchAgent.ts

@@ -112,8 +112,8 @@ const createBasicWolframAlphaSearchRetrieverChain = (llm: BaseChatModel) => {
       }
 
       const res = await searchSearxng(input, {
-        language: 'en',
-        engines: ['wolframalpha'],
+        lang: 'en',
+        engines: ['wolframalpha_api'],
       });
 
       const documents = res.results.map(

src/agents/youtubeSearchAgent.ts

@@ -113,8 +113,8 @@ const createBasicYoutubeSearchRetrieverChain = (llm: BaseChatModel) => {
       }
 
       const res = await searchSearxng(input, {
-        language: 'en',
-        engines: ['youtube'],
+        lang: 'en',
+        engines: ['youtube_api'],
       });
 
       const documents = res.results.map(

src/lib/searxng.ts

@@ -1,47 +1,19 @@
-import axios from 'axios';
 import { getSearxngApiEndpoint } from '../config';
 
-interface SearxngSearchOptions {
-  categories?: string[];
-  engines?: string[];
-  language?: string;
-  pageno?: number;
-}
+import { SearxngService, type SearxngSearchParameters } from 'searxng';
 
-interface SearxngSearchResult {
-  title: string;
-  url: string;
-  img_src?: string;
-  thumbnail_src?: string;
-  thumbnail?: string;
-  content?: string;
-  author?: string;
-  iframe_src?: string;
-}
+const searxng = new SearxngService({
+  baseURL: getSearxngApiEndpoint(),
+  defaultSearchParams: {
+    format: 'json'
+  }
+})
 
 export const searchSearxng = async (
   query: string,
-  opts?: SearxngSearchOptions,
+  opts?: SearxngSearchParameters,
 ) => {
-  const searxngURL = getSearxngApiEndpoint();
-
-  const url = new URL(`${searxngURL}/search?format=json`);
-  url.searchParams.append('q', query);
-
-  if (opts) {
-    Object.keys(opts).forEach((key) => {
-      if (Array.isArray(opts[key])) {
-        url.searchParams.append(key, opts[key].join(','));
-        return;
-      }
-      url.searchParams.append(key, opts[key]);
-    });
-  }
-
-  const res = await axios.get(url.toString());
-
-  const results: SearxngSearchResult[] = res.data.results;
-  const suggestions: string[] = res.data.suggestions;
 
+  const { results, suggestions } = await searxng.search(query, opts);
   return { results, suggestions };
 };

tsconfig.json

@@ -1,8 +1,8 @@
 {
   "compilerOptions": {
     "lib": ["ESNext"],
-    "module": "Node16",
-    "moduleResolution": "Node16",
+    "module": "ESNext",
+    "moduleResolution": "Node",
     "target": "ESNext",
     "outDir": "dist",
     "sourceMap": false,