Feature/add searchapi web browsing (#2224)

* Add SearchApi to web browsing

* UI modifications for SearchAPI

---------

Co-authored-by: Sebastjan Prachovskij <sebastjan.prachovskij@gmail.com>

.vscode/settings.json

@@ -41,6 +41,7 @@
     "Qdrant",
     "royalblue",
     "searxng",
+    "SearchApi",
     "Serper",
     "Serply",
     "streamable",

docker/.env.example

@@ -252,6 +252,10 @@ GID='1000'
 # AGENT_GSE_KEY=
 # AGENT_GSE_CTX=
 
+#------ SearchApi.io ----------- https://www.searchapi.io/
+# AGENT_SEARCHAPI_API_KEY=
+# AGENT_SEARCHAPI_ENGINE=google
+
 #------ Serper.dev ----------- https://serper.dev/
 # AGENT_SERPER_DEV_KEY=
 

docker/HOW_TO_USE_DOCKER.md

@@ -117,8 +117,8 @@ services:
       - WHISPER_PROVIDER=local
       - TTS_PROVIDER=native
       - PASSWORDMINCHAR=8
-      - AGENT_SERPER_DEV_KEY="SERPER DEV API KEY"
-      - AGENT_SERPLY_API_KEY="Serply.io API KEY"
+      # Add any other keys here for services or settings
+      # you can find in the docker/.env.example file
     volumes:
       - anythingllm_storage:/app/server/storage
     restart: always

frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx

@@ -50,6 +50,83 @@ export function GoogleSearchOptions({ settings }) {
   );
 }
 
+const SearchApiEngines = [
+  { name: "Google Search", value: "google" },
+  { name: "Google Maps", value: "google_maps" },
+  { name: "Google Shopping", value: "google_shopping" },
+  { name: "Google News", value: "google_news" },
+  { name: "Google Jobs", value: "google_jobs" },
+  { name: "Google Scholar", value: "google_scholar" },
+  { name: "Google Finance", value: "google_finance" },
+  { name: "Google Patents", value: "google_patents" },
+  { name: "YouTube", value: "youtube" },
+  { name: "Bing", value: "bing" },
+  { name: "Bing News", value: "bing_news" },
+  { name: "Amazon Product Search", value: "amazon_search" },
+  { name: "Baidu", value: "baidu" },
+];
+export function SearchApiOptions({ settings }) {
+  return (
+    <>
+      <p className="text-sm text-white/60 my-2">
+        You can get a free API key{" "}
+        <a
+          href="https://www.searchapi.io/"
+          target="_blank"
+          rel="noreferrer"
+          className="text-blue-300 underline"
+        >
+          from SearchApi.
+        </a>
+      </p>
+      <div className="flex gap-x-4">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            API Key
+          </label>
+          <input
+            type="password"
+            name="env::AgentSearchApiKey"
+            className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="SearchApi API Key"
+            defaultValue={settings?.AgentSearchApiKey ? "*".repeat(20) : ""}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            Engine
+          </label>
+          <select
+            name="env::AgentSearchApiEngine"
+            required={true}
+            className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            defaultValue={settings?.AgentSearchApiEngine || "google"}
+          >
+            {SearchApiEngines.map(({ name, value }) => (
+              <option key={name} value={value}>
+                {name}
+              </option>
+            ))}
+          </select>
+          {/* <input
+            type="text"
+            name="env::AgentSearchApiEngine"
+            className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="SearchApi engine (Google, Bing...)"
+            defaultValue={settings?.AgentSearchApiEngine || "google"}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          /> */}
+        </div>
+      </div>
+    </>
+  );
+}
+
 export function SerperDotDevOptions({ settings }) {
   return (
     <>

frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx

@@ -1,6 +1,7 @@
 import React, { useEffect, useRef, useState } from "react";
 import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
 import GoogleSearchIcon from "./icons/google.png";
+import SearchApiIcon from "./icons/searchapi.png";
 import SerperDotDevIcon from "./icons/serper.png";
 import BingSearchIcon from "./icons/bing.png";
 import SerplySearchIcon from "./icons/serply.png";
@@ -14,6 +15,7 @@ import {
 import SearchProviderItem from "./SearchProviderItem";
 import WebSearchImage from "@/media/agents/scrape-websites.png";
 import {
+  SearchApiOptions,
   SerperDotDevOptions,
   GoogleSearchOptions,
   BingSearchOptions,
@@ -38,6 +40,14 @@ const SEARCH_PROVIDERS = [
     description:
       "Web search powered by a custom Google Search Engine. Free for 100 queries per day.",
   },
+  {
+    name: "SearchApi",
+    value: "searchapi",
+    logo: SearchApiIcon,
+    options: (settings) => <SearchApiOptions settings={settings} />,
+    description:
+      "SearchApi delivers structured data from multiple search engines. Free for 100 queries, but then paid. ",
+  },
   {
     name: "Serper.dev",
     value: "serper-dot-dev",

server/.env.example

@@ -241,6 +241,10 @@ TTS_PROVIDER="native"
 # AGENT_GSE_KEY=
 # AGENT_GSE_CTX=
 
+#------ SearchApi.io ----------- https://www.searchapi.io/
+# AGENT_SEARCHAPI_API_KEY=
+# AGENT_SEARCHAPI_ENGINE=google
+
 #------ Serper.dev ----------- https://serper.dev/
 # AGENT_SERPER_DEV_KEY=
 

server/models/systemSettings.js

@@ -81,6 +81,7 @@ const SystemSettings = {
         if (
           ![
             "google-search-engine",
+            "searchapi",
             "serper-dot-dev",
             "bing-search",
             "serply-engine",
@@ -218,6 +219,8 @@ const SystemSettings = {
       // --------------------------------------------------------
       AgentGoogleSearchEngineId: process.env.AGENT_GSE_CTX || null,
       AgentGoogleSearchEngineKey: !!process.env.AGENT_GSE_KEY || null,
+      AgentSearchApiKey: !!process.env.AGENT_SEARCHAPI_API_KEY || null,
+      AgentSearchApiEngine: process.env.AGENT_SEARCHAPI_ENGINE || "google",
       AgentSerperApiKey: !!process.env.AGENT_SERPER_DEV_KEY || null,
       AgentBingSearchApiKey: !!process.env.AGENT_BING_SEARCH_API_KEY || null,
       AgentSerplyApiKey: !!process.env.AGENT_SERPLY_API_KEY || null,

server/utils/agents/aibitat/plugins/web-browsing.js

@@ -62,6 +62,9 @@ const webBrowsing = {
               case "google-search-engine":
                 engine = "_googleSearchEngine";
                 break;
+              case "searchapi":
+                engine = "_searchApi";
+                break;
               case "serper-dot-dev":
                 engine = "_serperDotDev";
                 break;
@@ -130,6 +133,72 @@ const webBrowsing = {
             return JSON.stringify(data);
           },
 
+          /**
+           * Use SearchApi
+           * SearchApi supports multiple search engines like Google Search, Bing Search, Baidu Search, Google News, YouTube, and many more.
+           * https://www.searchapi.io/
+           */
+          _searchApi: async function (query) {
+            if (!process.env.AGENT_SEARCHAPI_API_KEY) {
+              this.super.introspect(
+                `${this.caller}: I can't use SearchApi searching because the user has not defined the required API key.\nVisit: https://www.searchapi.io/ to create the API key for free.`
+              );
+              return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`;
+            }
+
+            this.super.introspect(
+              `${this.caller}: Using SearchApi to search for "${
+                query.length > 100 ? `${query.slice(0, 100)}...` : query
+              }"`
+            );
+
+            const engine = process.env.AGENT_SEARCHAPI_ENGINE;
+            const params = new URLSearchParams({
+              engine: engine,
+              q: query,
+            });
+
+            const url = `https://www.searchapi.io/api/v1/search?${params.toString()}`;
+            const { response, error } = await fetch(url, {
+              method: "GET",
+              headers: {
+                Authorization: `Bearer ${process.env.AGENT_SEARCHAPI_API_KEY}`,
+                "Content-Type": "application/json",
+                "X-SearchApi-Source": "AnythingLLM",
+              },
+            })
+              .then((res) => res.json())
+              .then((data) => {
+                return { response: data, error: null };
+              })
+              .catch((e) => {
+                return { response: null, error: e.message };
+              });
+            if (error)
+              return `There was an error searching for content. ${error}`;
+
+            const data = [];
+            if (response.hasOwnProperty("knowledge_graph"))
+              data.push(response.knowledge_graph?.description);
+            if (response.hasOwnProperty("answer_box"))
+              data.push(response.answer_box?.answer);
+            response.organic_results?.forEach((searchResult) => {
+              const { title, link, snippet } = searchResult;
+              data.push({
+                title,
+                link,
+                snippet,
+              });
+            });
+
+            if (data.length === 0)
+              return `No information was found online for the search query.`;
+            this.super.introspect(
+              `${this.caller}: I found ${data.length} results - looking over them now.`
+            );
+            return JSON.stringify(data);
+          },
+
           /**
            * Use Serper.dev
            * Free to set up, easy to use, 2,500 calls for free one-time

server/utils/helpers/updateENV.js

@@ -435,6 +435,14 @@ const KEY_MAPPING = {
     envKey: "AGENT_GSE_KEY",
     checks: [],
   },
+  AgentSearchApiKey: {
+    envKey: "AGENT_SEARCHAPI_API_KEY",
+    checks: [],
+  },
+  AgentSearchApiEngine: {
+    envKey: "AGENT_SEARCHAPI_ENGINE",
+    checks: [],
+  },
   AgentSerperApiKey: {
     envKey: "AGENT_SERPER_DEV_KEY",
     checks: [],