From 9d827d4cc2e10716c565119e7442ddb693e93483 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Mon, 24 Jun 2024 21:34:14 +0530
Subject: [PATCH 01/37] feat(package): update WS module

---
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index ce5f0eb..612da3c 100644
--- a/package.json
+++ b/package.json
@@ -31,7 +31,7 @@
     "express": "^4.19.2",
     "langchain": "^0.1.30",
     "winston": "^3.13.0",
-    "ws": "^8.16.0",
+    "ws": "^8.17.1",
     "zod": "^3.22.4"
   }
 }
diff --git a/yarn.lock b/yarn.lock
index 6ec0ff2..a6636fb 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2054,10 +2054,10 @@ wrappy@1:
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
 
-ws@^8.16.0:
-  version "8.16.0"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-8.16.0.tgz#d1cd774f36fbc07165066a60e40323eab6446fd4"
-  integrity sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==
+ws@^8.17.1:
+  version "8.17.1"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b"
+  integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==
 
 yallist@^4.0.0:
   version "4.0.0"

From a152e581322745037f64df7648467ceb1955e613 Mon Sep 17 00:00:00 2001
From: asifrahaman13 <asifrahaman162@gmail.com>
Date: Tue, 25 Jun 2024 15:43:36 +0530
Subject: [PATCH 02/37] =?UTF-8?q?=F0=9F=8E=89=20wip:=20implemented=20error?=
 =?UTF-8?q?=20state=20for=20backend=20socket=20connection=20and=20othe?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 ui/components/ChatWindow.tsx | 67 +++++++++++++++++++++++++-----------
 1 file changed, 46 insertions(+), 21 deletions(-)

diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx
index 33bc3fb..728f419 100644
--- a/ui/components/ChatWindow.tsx
+++ b/ui/components/ChatWindow.tsx
@@ -18,7 +18,11 @@ export type Message = {
   sources?: Document[];
 };
 
-const useSocket = (url: string, setIsReady: (ready: boolean) => void) => {
+const useSocket = (
+  url: string,
+  setIsReady: (ready: boolean) => void,
+  setError: (error: boolean) => void,
+) => {
   const [ws, setWs] = useState<WebSocket | null>(null);
 
   useEffect(() => {
@@ -102,28 +106,36 @@ const useSocket = (url: string, setIsReady: (ready: boolean) => void) => {
 
         const ws = new WebSocket(wsURL.toString());
 
+        const timeoutId = setTimeout(() => {
+          if (ws.readyState !== 1) {
+            ws.close();
+            setError(true);
+            toast.error(
+              'Failed to connect to the server. Please try again later.',
+            );
+          }
+        }, 10000); // 10 seconds timeout
+
         ws.onopen = () => {
           console.log('[DEBUG] open');
+          clearTimeout(timeoutId);
+          setError(false);
+          setIsReady(true);
         };
 
-        const stateCheckInterval = setInterval(() => {
-          if (ws.readyState === 1) {
-            setIsReady(true);
-            clearInterval(stateCheckInterval);
-          }
-        }, 100);
+        ws.onerror = () => {
+          clearTimeout(timeoutId);
+          setError(true);
+          toast.error('WebSocket connection error.');
+        };
+
+        ws.onclose = () => {
+          clearTimeout(timeoutId);
+          setError(true);
+          console.log('[DEBUG] closed');
+        };
 
         setWs(ws);
-
-        ws.onmessage = (e) => {
-          const parsedData = JSON.parse(e.data);
-          if (parsedData.type === 'error') {
-            toast.error(parsedData.data);
-            if (parsedData.key === 'INVALID_MODEL_SELECTED') {
-              localStorage.clear();
-            }
-          }
-        };
       };
 
       connectWs();
@@ -133,7 +145,7 @@ const useSocket = (url: string, setIsReady: (ready: boolean) => void) => {
       ws?.close();
       console.log('[DEBUG] closed');
     };
-  }, [ws, url, setIsReady]);
+  }, [ws, url, setIsReady, setError]);
 
   return ws;
 };
@@ -143,7 +155,12 @@ const ChatWindow = () => {
   const initialMessage = searchParams.get('q');
 
   const [isReady, setIsReady] = useState(false);
-  const ws = useSocket(process.env.NEXT_PUBLIC_WS_URL!, setIsReady);
+  const [hasError, setHasError] = useState(false);
+  const ws = useSocket(
+    process.env.NEXT_PUBLIC_WS_URL!,
+    setIsReady,
+    setHasError,
+  );
 
   const [chatHistory, setChatHistory] = useState<[string, string][]>([]);
   const [messages, setMessages] = useState<Message[]>([]);
@@ -298,6 +315,14 @@ const ChatWindow = () => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [isReady, initialMessage]);
 
+  if (hasError) {
+    return (
+      <div className="flex flex-col items-center justify-center min-h-screen">
+        <p>Failed to connect to the server. Please try again later.</p>
+      </div>
+    );
+  }
+
   return isReady ? (
     <div>
       {messages.length > 0 ? (
@@ -329,11 +354,11 @@ const ChatWindow = () => {
         xmlns="http://www.w3.org/2000/svg"
       >
         <path
-          d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
+          d="M100 50.5908C100.003 78.2051 78.1951 100.003 50.5908 100C22.9765 99.9972 0.997224 78.018 1 50.4037C1.00281 22.7993 22.8108 0.997224 50.4251 1C78.0395 1.00281 100.018 22.8108 100 50.4251ZM9.08164 50.594C9.06312 73.3997 27.7909 92.1272 50.5966 92.1457C73.4023 92.1642 92.1298 73.4365 92.1483 50.6308C92.1669 27.8251 73.4392 9.0973 50.6335 9.07878C27.8278 9.06026 9.10003 27.787 9.08164 50.594Z"
           fill="currentColor"
         />
         <path
-          d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
+          d="M93.9676 39.0409C96.393 38.4037 97.8624 35.9116 96.9801 33.5533C95.1945 28.8227 92.871 24.3692 90.0681 20.348C85.6237 14.1775 79.4473 9.36872 72.0454 6.45794C64.6435 3.54717 56.3134 2.65431 48.3133 3.89319C45.869 4.27179 44.3768 6.77534 45.014 9.20079C45.6512 11.6262 48.1343 13.0956 50.5786 12.717C56.5073 11.8281 62.5542 12.5399 68.0406 14.7911C73.527 17.0422 78.2187 20.7487 81.5841 25.4923C83.7976 28.5886 85.4467 32.059 86.4416 35.7474C87.1273 38.1189 89.5423 39.6781 91.9676 39.0409Z"
           fill="currentFill"
         />
       </svg>

From 4545ff1d7d2004db365ce869d373e350c9a95979 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Tue, 25 Jun 2024 16:11:39 +0530
Subject: [PATCH 03/37] feat(chat-window): adjust color & size

---
 README.md                    | 2 ++
 ui/components/ChatWindow.tsx | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 77c964f..6e07f54 100644
--- a/README.md
+++ b/README.md
@@ -101,6 +101,7 @@ If you're encountering an Ollama connection error, it is likely due to the backe
 
 1. **Check your Ollama API URL:** Ensure that the API URL is correctly set in the settings menu.
 2. **Update API URL Based on OS:**
+
    - **Windows:** Use `http://host.docker.internal:11434`
    - **Mac:** Use `http://host.docker.internal:11434`
    - **Linux:** Use `http://<private_ip_of_host>:11434`
@@ -108,6 +109,7 @@ If you're encountering an Ollama connection error, it is likely due to the backe
    Adjust the port number if you're using a different one.
 
 3. **Linux Users - Expose Ollama to Network:**
+
    - Serve Ollama over your network with the command:
 
      ```bash
diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx
index 728f419..35ffd1e 100644
--- a/ui/components/ChatWindow.tsx
+++ b/ui/components/ChatWindow.tsx
@@ -114,7 +114,7 @@ const useSocket = (
               'Failed to connect to the server. Please try again later.',
             );
           }
-        }, 10000); // 10 seconds timeout
+        }, 10000);
 
         ws.onopen = () => {
           console.log('[DEBUG] open');
@@ -318,7 +318,9 @@ const ChatWindow = () => {
   if (hasError) {
     return (
       <div className="flex flex-col items-center justify-center min-h-screen">
-        <p>Failed to connect to the server. Please try again later.</p>
+        <p className="dark:text-white/70 text-black/70 text-sm">
+          Failed to connect to the server. Please try again later.
+        </p>
       </div>
     );
   }

From 91bba8eaca2a8c72807a5d4d6285b9a74bf7c9bf Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Fri, 28 Jun 2024 09:34:03 +0530
Subject: [PATCH 04/37] feat(utils): accept string in time difference

---
 ui/lib/utils.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ui/lib/utils.ts b/ui/lib/utils.ts
index 6b35b90..30d6da5 100644
--- a/ui/lib/utils.ts
+++ b/ui/lib/utils.ts
@@ -3,7 +3,13 @@ import { twMerge } from 'tailwind-merge';
 
 export const cn = (...classes: ClassValue[]) => twMerge(clsx(...classes));
 
-export const formatTimeDifference = (date1: Date, date2: Date): string => {
+export const formatTimeDifference = (
+  date1: Date | string,
+  date2: Date | string,
+): string => {
+  date1 = new Date(date1);
+  date2 = new Date(date2);
+
   const diffInSeconds = Math.floor(
     Math.abs(date2.getTime() - date1.getTime()) / 1000,
   );

From f7d1364f30d5e9b1c46a147b9818d7df0e251bb5 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Fri, 28 Jun 2024 09:34:40 +0530
Subject: [PATCH 05/37] feat(discover): remove unadded page

---
 ui/app/discover/page.tsx | 5 -----
 1 file changed, 5 deletions(-)
 delete mode 100644 ui/app/discover/page.tsx

diff --git a/ui/app/discover/page.tsx b/ui/app/discover/page.tsx
deleted file mode 100644
index a443a17..0000000
--- a/ui/app/discover/page.tsx
+++ /dev/null
@@ -1,5 +0,0 @@
-const Page = () => {
-  return <div>page</div>;
-};
-
-export default Page;

From 7879167b13a478d266545c296bc02991cb842049 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:07:52 +0530
Subject: [PATCH 06/37] feat(package): add better-sqlite3

---
 package.json |   7 +-
 yarn.lock    | 383 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 389 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 612da3c..2f6f7f2 100644
--- a/package.json
+++ b/package.json
@@ -4,16 +4,19 @@
   "license": "MIT",
   "author": "ItzCrazyKns",
   "scripts": {
-    "start": "node dist/app.js",
+    "start": "npm run db:push && node dist/app.js",
     "build": "tsc",
     "dev": "nodemon src/app.ts",
+    "db:push": "drizzle-kit push sqlite",
     "format": "prettier . --check",
     "format:write": "prettier . --write"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.10",
     "@types/cors": "^2.8.17",
     "@types/express": "^4.17.21",
     "@types/readable-stream": "^4.0.11",
+    "drizzle-kit": "^0.22.7",
     "nodemon": "^3.1.0",
     "prettier": "^3.2.5",
     "ts-node": "^10.9.2",
@@ -24,10 +27,12 @@
     "@langchain/openai": "^0.0.25",
     "@xenova/transformers": "^2.17.1",
     "axios": "^1.6.8",
+    "better-sqlite3": "^11.0.0",
     "compute-cosine-similarity": "^1.1.0",
     "compute-dot": "^1.1.0",
     "cors": "^2.8.5",
     "dotenv": "^16.4.5",
+    "drizzle-orm": "^0.31.2",
     "express": "^4.19.2",
     "langchain": "^0.1.30",
     "winston": "^3.13.0",
diff --git a/yarn.lock b/yarn.lock
index a6636fb..d418edf 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -38,6 +38,247 @@
     enabled "2.0.x"
     kuler "^2.0.0"
 
+"@esbuild-kit/core-utils@^3.3.2":
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/@esbuild-kit/core-utils/-/core-utils-3.3.2.tgz#186b6598a5066f0413471d7c4d45828e399ba96c"
+  integrity sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==
+  dependencies:
+    esbuild "~0.18.20"
+    source-map-support "^0.5.21"
+
+"@esbuild-kit/esm-loader@^2.5.5":
+  version "2.6.5"
+  resolved "https://registry.yarnpkg.com/@esbuild-kit/esm-loader/-/esm-loader-2.6.5.tgz#6eedee46095d7d13b1efc381e2211ed1c60e64ea"
+  integrity sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==
+  dependencies:
+    "@esbuild-kit/core-utils" "^3.3.2"
+    get-tsconfig "^4.7.0"
+
+"@esbuild/aix-ppc64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.19.12.tgz#d1bc06aedb6936b3b6d313bf809a5a40387d2b7f"
+  integrity sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA==
+
+"@esbuild/android-arm64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz#984b4f9c8d0377443cc2dfcef266d02244593622"
+  integrity sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==
+
+"@esbuild/android-arm64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.19.12.tgz#7ad65a36cfdb7e0d429c353e00f680d737c2aed4"
+  integrity sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA==
+
+"@esbuild/android-arm@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.18.20.tgz#fedb265bc3a589c84cc11f810804f234947c3682"
+  integrity sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==
+
+"@esbuild/android-arm@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.19.12.tgz#b0c26536f37776162ca8bde25e42040c203f2824"
+  integrity sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w==
+
+"@esbuild/android-x64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.18.20.tgz#35cf419c4cfc8babe8893d296cd990e9e9f756f2"
+  integrity sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==
+
+"@esbuild/android-x64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.19.12.tgz#cb13e2211282012194d89bf3bfe7721273473b3d"
+  integrity sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew==
+
+"@esbuild/darwin-arm64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz#08172cbeccf95fbc383399a7f39cfbddaeb0d7c1"
+  integrity sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==
+
+"@esbuild/darwin-arm64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.19.12.tgz#cbee41e988020d4b516e9d9e44dd29200996275e"
+  integrity sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g==
+
+"@esbuild/darwin-x64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz#d70d5790d8bf475556b67d0f8b7c5bdff053d85d"
+  integrity sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==
+
+"@esbuild/darwin-x64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.19.12.tgz#e37d9633246d52aecf491ee916ece709f9d5f4cd"
+  integrity sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A==
+
+"@esbuild/freebsd-arm64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz#98755cd12707f93f210e2494d6a4b51b96977f54"
+  integrity sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==
+
+"@esbuild/freebsd-arm64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.12.tgz#1ee4d8b682ed363b08af74d1ea2b2b4dbba76487"
+  integrity sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA==
+
+"@esbuild/freebsd-x64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz#c1eb2bff03915f87c29cece4c1a7fa1f423b066e"
+  integrity sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==
+
+"@esbuild/freebsd-x64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.19.12.tgz#37a693553d42ff77cd7126764b535fb6cc28a11c"
+  integrity sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg==
+
+"@esbuild/linux-arm64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz#bad4238bd8f4fc25b5a021280c770ab5fc3a02a0"
+  integrity sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==
+
+"@esbuild/linux-arm64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.19.12.tgz#be9b145985ec6c57470e0e051d887b09dddb2d4b"
+  integrity sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA==
+
+"@esbuild/linux-arm@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz#3e617c61f33508a27150ee417543c8ab5acc73b0"
+  integrity sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==
+
+"@esbuild/linux-arm@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.19.12.tgz#207ecd982a8db95f7b5279207d0ff2331acf5eef"
+  integrity sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w==
+
+"@esbuild/linux-ia32@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz#699391cccba9aee6019b7f9892eb99219f1570a7"
+  integrity sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==
+
+"@esbuild/linux-ia32@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.19.12.tgz#d0d86b5ca1562523dc284a6723293a52d5860601"
+  integrity sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA==
+
+"@esbuild/linux-loong64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz#e6fccb7aac178dd2ffb9860465ac89d7f23b977d"
+  integrity sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==
+
+"@esbuild/linux-loong64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.19.12.tgz#9a37f87fec4b8408e682b528391fa22afd952299"
+  integrity sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA==
+
+"@esbuild/linux-mips64el@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz#eeff3a937de9c2310de30622a957ad1bd9183231"
+  integrity sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==
+
+"@esbuild/linux-mips64el@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.19.12.tgz#4ddebd4e6eeba20b509d8e74c8e30d8ace0b89ec"
+  integrity sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w==
+
+"@esbuild/linux-ppc64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz#2f7156bde20b01527993e6881435ad79ba9599fb"
+  integrity sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==
+
+"@esbuild/linux-ppc64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.19.12.tgz#adb67dadb73656849f63cd522f5ecb351dd8dee8"
+  integrity sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg==
+
+"@esbuild/linux-riscv64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz#6628389f210123d8b4743045af8caa7d4ddfc7a6"
+  integrity sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==
+
+"@esbuild/linux-riscv64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.19.12.tgz#11bc0698bf0a2abf8727f1c7ace2112612c15adf"
+  integrity sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg==
+
+"@esbuild/linux-s390x@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz#255e81fb289b101026131858ab99fba63dcf0071"
+  integrity sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==
+
+"@esbuild/linux-s390x@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.19.12.tgz#e86fb8ffba7c5c92ba91fc3b27ed5a70196c3cc8"
+  integrity sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg==
+
+"@esbuild/linux-x64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz#c7690b3417af318a9b6f96df3031a8865176d338"
+  integrity sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==
+
+"@esbuild/linux-x64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.19.12.tgz#5f37cfdc705aea687dfe5dfbec086a05acfe9c78"
+  integrity sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg==
+
+"@esbuild/netbsd-x64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz#30e8cd8a3dded63975e2df2438ca109601ebe0d1"
+  integrity sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==
+
+"@esbuild/netbsd-x64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.19.12.tgz#29da566a75324e0d0dd7e47519ba2f7ef168657b"
+  integrity sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA==
+
+"@esbuild/openbsd-x64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz#7812af31b205055874c8082ea9cf9ab0da6217ae"
+  integrity sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==
+
+"@esbuild/openbsd-x64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.19.12.tgz#306c0acbdb5a99c95be98bdd1d47c916e7dc3ff0"
+  integrity sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw==
+
+"@esbuild/sunos-x64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz#d5c275c3b4e73c9b0ecd38d1ca62c020f887ab9d"
+  integrity sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==
+
+"@esbuild/sunos-x64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.19.12.tgz#0933eaab9af8b9b2c930236f62aae3fc593faf30"
+  integrity sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA==
+
+"@esbuild/win32-arm64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz#73bc7f5a9f8a77805f357fab97f290d0e4820ac9"
+  integrity sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==
+
+"@esbuild/win32-arm64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.19.12.tgz#773bdbaa1971b36db2f6560088639ccd1e6773ae"
+  integrity sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A==
+
+"@esbuild/win32-ia32@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz#ec93cbf0ef1085cc12e71e0d661d20569ff42102"
+  integrity sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==
+
+"@esbuild/win32-ia32@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.19.12.tgz#000516cad06354cc84a73f0943a4aa690ef6fd67"
+  integrity sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ==
+
+"@esbuild/win32-x64@0.18.20":
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz#786c5f41f043b07afb1af37683d7c33668858f6d"
+  integrity sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==
+
+"@esbuild/win32-x64@0.19.12":
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.19.12.tgz#c57c8afbb4054a3ab8317591a0b7320360b444ae"
+  integrity sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA==
+
 "@huggingface/jinja@^0.2.2":
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/@huggingface/jinja/-/jinja-0.2.2.tgz#faeb205a9d6995089bef52655ddd8245d3190627"
@@ -180,6 +421,13 @@
   resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9"
   integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==
 
+"@types/better-sqlite3@^7.6.10":
+  version "7.6.10"
+  resolved "https://registry.yarnpkg.com/@types/better-sqlite3/-/better-sqlite3-7.6.10.tgz#1818e56490953404acfd44cdde0464f201be6105"
+  integrity sha512-TZBjD+yOsyrUJGmcUj6OS3JADk3+UZcNv3NOBqGkM09bZdi28fNZw8ODqbMOLfKCu7RYCO62/ldq1iHbzxqoPw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/body-parser@*":
   version "1.19.5"
   resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.5.tgz#04ce9a3b677dc8bd681a17da1ab9835dc9d3ede4"
@@ -464,6 +712,14 @@ base64-js@^1.3.1, base64-js@^1.5.1:
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
 
+better-sqlite3@^11.0.0:
+  version "11.0.0"
+  resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-11.0.0.tgz#12083acfe0ded6abdba908ed73520f2003e3ea0e"
+  integrity sha512-1NnNhmT3EZTsKtofJlMox1jkMxdedILury74PwUbQBjWgo4tL4kf7uTAjU55mgQwjdzqakSTjkf+E1imrFwjnA==
+  dependencies:
+    bindings "^1.5.0"
+    prebuild-install "^7.1.1"
+
 binary-extensions@^2.0.0, binary-extensions@^2.2.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522"
@@ -474,6 +730,13 @@ binary-search@^1.3.5:
   resolved "https://registry.yarnpkg.com/binary-search/-/binary-search-1.3.6.tgz#e32426016a0c5092f0f3598836a1c7da3560565c"
   integrity sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==
 
+bindings@^1.5.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
+  integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==
+  dependencies:
+    file-uri-to-path "1.0.0"
+
 bl@^4.0.3:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
@@ -516,6 +779,11 @@ braces@~3.0.2:
   dependencies:
     fill-range "^7.0.1"
 
+buffer-from@^1.0.0:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
+  integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
+
 buffer@^5.5.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
@@ -723,6 +991,13 @@ debug@^4:
   dependencies:
     ms "2.1.2"
 
+debug@^4.3.4:
+  version "4.3.5"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e"
+  integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==
+  dependencies:
+    ms "2.1.2"
+
 decamelize@1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@@ -787,6 +1062,20 @@ dotenv@^16.4.5:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f"
   integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==
 
+drizzle-kit@^0.22.7:
+  version "0.22.7"
+  resolved "https://registry.yarnpkg.com/drizzle-kit/-/drizzle-kit-0.22.7.tgz#4339c3e24c6555ea8cbad605f005b3db3e604a9c"
+  integrity sha512-9THPCb2l1GPt7wxhws9LvTR0YG565ZlVgTuqGMwjs590Kch1pXu4GyjEArVijSF5m0OBj3qgdeKmuJXhKXgWFw==
+  dependencies:
+    "@esbuild-kit/esm-loader" "^2.5.5"
+    esbuild "^0.19.7"
+    esbuild-register "^3.5.0"
+
+drizzle-orm@^0.31.2:
+  version "0.31.2"
+  resolved "https://registry.yarnpkg.com/drizzle-orm/-/drizzle-orm-0.31.2.tgz#221a257dd487bab49ddb88a17bd82388600cf655"
+  integrity sha512-QnenevbnnAzmbNzQwbhklvIYrDE8YER8K7kSrAWQSV1YvFCdSQPzj+jzqRdTSsV2cDqSpQ0NXGyL1G9I43LDLg==
+
 ee-first@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
@@ -821,6 +1110,70 @@ es-errors@^1.3.0:
   resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f"
   integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==
 
+esbuild-register@^3.5.0:
+  version "3.5.0"
+  resolved "https://registry.yarnpkg.com/esbuild-register/-/esbuild-register-3.5.0.tgz#449613fb29ab94325c722f560f800dd946dc8ea8"
+  integrity sha512-+4G/XmakeBAsvJuDugJvtyF1x+XJT4FMocynNpxrvEBViirpfUn2PgNpCHedfWhF4WokNsO/OvMKrmJOIJsI5A==
+  dependencies:
+    debug "^4.3.4"
+
+esbuild@^0.19.7:
+  version "0.19.12"
+  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.19.12.tgz#dc82ee5dc79e82f5a5c3b4323a2a641827db3e04"
+  integrity sha512-aARqgq8roFBj054KvQr5f1sFu0D65G+miZRCuJyJ0G13Zwx7vRar5Zhn2tkQNzIXcBrNVsv/8stehpj+GAjgbg==
+  optionalDependencies:
+    "@esbuild/aix-ppc64" "0.19.12"
+    "@esbuild/android-arm" "0.19.12"
+    "@esbuild/android-arm64" "0.19.12"
+    "@esbuild/android-x64" "0.19.12"
+    "@esbuild/darwin-arm64" "0.19.12"
+    "@esbuild/darwin-x64" "0.19.12"
+    "@esbuild/freebsd-arm64" "0.19.12"
+    "@esbuild/freebsd-x64" "0.19.12"
+    "@esbuild/linux-arm" "0.19.12"
+    "@esbuild/linux-arm64" "0.19.12"
+    "@esbuild/linux-ia32" "0.19.12"
+    "@esbuild/linux-loong64" "0.19.12"
+    "@esbuild/linux-mips64el" "0.19.12"
+    "@esbuild/linux-ppc64" "0.19.12"
+    "@esbuild/linux-riscv64" "0.19.12"
+    "@esbuild/linux-s390x" "0.19.12"
+    "@esbuild/linux-x64" "0.19.12"
+    "@esbuild/netbsd-x64" "0.19.12"
+    "@esbuild/openbsd-x64" "0.19.12"
+    "@esbuild/sunos-x64" "0.19.12"
+    "@esbuild/win32-arm64" "0.19.12"
+    "@esbuild/win32-ia32" "0.19.12"
+    "@esbuild/win32-x64" "0.19.12"
+
+esbuild@~0.18.20:
+  version "0.18.20"
+  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.18.20.tgz#4709f5a34801b43b799ab7d6d82f7284a9b7a7a6"
+  integrity sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==
+  optionalDependencies:
+    "@esbuild/android-arm" "0.18.20"
+    "@esbuild/android-arm64" "0.18.20"
+    "@esbuild/android-x64" "0.18.20"
+    "@esbuild/darwin-arm64" "0.18.20"
+    "@esbuild/darwin-x64" "0.18.20"
+    "@esbuild/freebsd-arm64" "0.18.20"
+    "@esbuild/freebsd-x64" "0.18.20"
+    "@esbuild/linux-arm" "0.18.20"
+    "@esbuild/linux-arm64" "0.18.20"
+    "@esbuild/linux-ia32" "0.18.20"
+    "@esbuild/linux-loong64" "0.18.20"
+    "@esbuild/linux-mips64el" "0.18.20"
+    "@esbuild/linux-ppc64" "0.18.20"
+    "@esbuild/linux-riscv64" "0.18.20"
+    "@esbuild/linux-s390x" "0.18.20"
+    "@esbuild/linux-x64" "0.18.20"
+    "@esbuild/netbsd-x64" "0.18.20"
+    "@esbuild/openbsd-x64" "0.18.20"
+    "@esbuild/sunos-x64" "0.18.20"
+    "@esbuild/win32-arm64" "0.18.20"
+    "@esbuild/win32-ia32" "0.18.20"
+    "@esbuild/win32-x64" "0.18.20"
+
 escape-html@~1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
@@ -898,6 +1251,11 @@ fecha@^4.2.0:
   resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd"
   integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==
 
+file-uri-to-path@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
+  integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==
+
 fill-range@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
@@ -996,6 +1354,13 @@ get-intrinsic@^1.1.3, get-intrinsic@^1.2.4:
     has-symbols "^1.0.3"
     hasown "^2.0.0"
 
+get-tsconfig@^4.7.0:
+  version "4.7.5"
+  resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.7.5.tgz#5e012498579e9a6947511ed0cd403272c7acbbaf"
+  integrity sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==
+  dependencies:
+    resolve-pkg-maps "^1.0.0"
+
 github-from-package@0.0.0:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce"
@@ -1667,6 +2032,11 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
+resolve-pkg-maps@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f"
+  integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==
+
 retry@^0.13.1:
   version "0.13.1"
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658"
@@ -1797,6 +2167,19 @@ simple-update-notifier@^2.0.0:
   dependencies:
     semver "^7.5.3"
 
+source-map-support@^0.5.21:
+  version "0.5.21"
+  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
+  integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==
+  dependencies:
+    buffer-from "^1.0.0"
+    source-map "^0.6.0"
+
+source-map@^0.6.0:
+  version "0.6.1"
+  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
+  integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
+
 stack-trace@0.0.x:
   version "0.0.10"
   resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"

From 93b90dc1c4f7a38f0540c45531349ca2998b31b6 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:08:11 +0530
Subject: [PATCH 07/37] feat(db): create schema & config files

---
 drizzle.config.ts | 10 ++++++++++
 src/db/index.ts   | 10 ++++++++++
 src/db/schema.ts  | 19 +++++++++++++++++++
 3 files changed, 39 insertions(+)
 create mode 100644 drizzle.config.ts
 create mode 100644 src/db/index.ts
 create mode 100644 src/db/schema.ts

diff --git a/drizzle.config.ts b/drizzle.config.ts
new file mode 100644
index 0000000..9ac3ec5
--- /dev/null
+++ b/drizzle.config.ts
@@ -0,0 +1,10 @@
+import { defineConfig } from 'drizzle-kit';
+
+export default defineConfig({
+  dialect: 'sqlite',
+  schema: './src/db/schema.ts',
+  out: './drizzle',
+  dbCredentials: {
+    url: './data/db.sqlite',
+  },
+});
diff --git a/src/db/index.ts b/src/db/index.ts
new file mode 100644
index 0000000..b431b47
--- /dev/null
+++ b/src/db/index.ts
@@ -0,0 +1,10 @@
+import { drizzle } from 'drizzle-orm/better-sqlite3';
+import Database from 'better-sqlite3';
+import * as schema from './schema';
+
+const sqlite = new Database('data/db.sqlite');
+const db = drizzle(sqlite, {
+  schema: schema,
+});
+
+export default db;
diff --git a/src/db/schema.ts b/src/db/schema.ts
new file mode 100644
index 0000000..9eefa55
--- /dev/null
+++ b/src/db/schema.ts
@@ -0,0 +1,19 @@
+import { text, integer, sqliteTable } from 'drizzle-orm/sqlite-core';
+
+export const messages = sqliteTable('messages', {
+  id: integer('id').primaryKey(),
+  content: text('content').notNull(),
+  chatId: text('chatId').notNull(),
+  messageId: text('messageId').notNull(),
+  role: text('type', { enum: ['assistant', 'user'] }),
+  metadata: text('metadata', {
+    mode: 'json',
+  }),
+});
+
+export const chats = sqliteTable('chats', {
+  id: text('id').primaryKey(),
+  title: text('title').notNull(),
+  createdAt: text('createdAt').notNull(),
+  focusMode: text('focusMode').notNull(),
+});

From d806c7e5813fd801ba4259f84ebb0d2657bb6e54 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:09:13 +0530
Subject: [PATCH 08/37] feat(app): add chats route

---
 src/routes/chats.ts | 45 +++++++++++++++++++++++++++++++++++++++++++++
 src/routes/index.ts |  2 ++
 2 files changed, 47 insertions(+)
 create mode 100644 src/routes/chats.ts

diff --git a/src/routes/chats.ts b/src/routes/chats.ts
new file mode 100644
index 0000000..f970ba4
--- /dev/null
+++ b/src/routes/chats.ts
@@ -0,0 +1,45 @@
+import express from 'express';
+import logger from '../utils/logger';
+import db from '../db/index';
+import { eq } from 'drizzle-orm';
+import { chats, messages } from '../db/schema';
+
+const router = express.Router();
+
+router.get('/', async (_, res) => {
+  try {
+    let chats = await db.query.chats.findMany();
+
+    /* Reorder the chats (in opposite direction) */
+
+    chats = chats.reverse();
+
+    return res.status(200).json({ chats: chats });
+  } catch (err) {
+    res.status(500).json({ message: 'An error has occurred.' });
+    logger.error(`Error in getting chats: ${err.message}`);
+  }
+});
+
+router.get('/:id', async (req, res) => {
+  try {
+    const chatExists = await db.query.chats.findFirst({
+      where: eq(chats.id, req.params.id),
+    });
+
+    if (!chatExists) {
+      return res.status(404).json({ message: 'Chat not found' });
+    }
+
+    const chatMessages = await db.query.messages.findMany({
+      where: eq(messages.chatId, req.params.id),
+    });
+
+    return res.status(200).json({ chat: chatExists, messages: chatMessages });
+  } catch (err) {
+    res.status(500).json({ message: 'An error has occurred.' });
+    logger.error(`Error in getting chat: ${err.message}`);
+  }
+});
+
+export default router;
diff --git a/src/routes/index.ts b/src/routes/index.ts
index 257e677..af928ab 100644
--- a/src/routes/index.ts
+++ b/src/routes/index.ts
@@ -4,6 +4,7 @@ import videosRouter from './videos';
 import configRouter from './config';
 import modelsRouter from './models';
 import suggestionsRouter from './suggestions';
+import chatsRouter from './chats';
 
 const router = express.Router();
 
@@ -12,5 +13,6 @@ router.use('/videos', videosRouter);
 router.use('/config', configRouter);
 router.use('/models', modelsRouter);
 router.use('/suggestions', suggestionsRouter);
+router.use('/chats', chatsRouter);
 
 export default router;

From 61044715e97c4d47343284e6d63b4d96b4c00972 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:09:31 +0530
Subject: [PATCH 09/37] feat(msg-handler): update message types

---
 src/websocket/messageHandler.ts | 84 ++++++++++++++++++++++++++++-----
 1 file changed, 73 insertions(+), 11 deletions(-)

diff --git a/src/websocket/messageHandler.ts b/src/websocket/messageHandler.ts
index 98f67c2..0afda9f 100644
--- a/src/websocket/messageHandler.ts
+++ b/src/websocket/messageHandler.ts
@@ -9,11 +9,21 @@ import handleRedditSearch from '../agents/redditSearchAgent';
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 import type { Embeddings } from '@langchain/core/embeddings';
 import logger from '../utils/logger';
+import db from '../db';
+import { chats, messages } from '../db/schema';
+import { eq } from 'drizzle-orm';
+import crypto from 'crypto';
 
 type Message = {
-  type: string;
+  messageId: string;
+  chatId: string;
   content: string;
+};
+
+type WSMessage = {
+  message: Message;
   copilot: boolean;
+  type: string;
   focusMode: string;
   history: Array<[string, string]>;
 };
@@ -30,8 +40,12 @@ const searchHandlers = {
 const handleEmitterEvents = (
   emitter: EventEmitter,
   ws: WebSocket,
-  id: string,
+  messageId: string,
+  chatId: string,
 ) => {
+  let recievedMessage = '';
+  let sources = [];
+
   emitter.on('data', (data) => {
     const parsedData = JSON.parse(data);
     if (parsedData.type === 'response') {
@@ -39,21 +53,36 @@ const handleEmitterEvents = (
         JSON.stringify({
           type: 'message',
           data: parsedData.data,
-          messageId: id,
+          messageId: messageId,
         }),
       );
+      recievedMessage += parsedData.data;
     } else if (parsedData.type === 'sources') {
       ws.send(
         JSON.stringify({
           type: 'sources',
           data: parsedData.data,
-          messageId: id,
+          messageId: messageId,
         }),
       );
+      sources = parsedData.data;
     }
   });
   emitter.on('end', () => {
-    ws.send(JSON.stringify({ type: 'messageEnd', messageId: id }));
+    ws.send(JSON.stringify({ type: 'messageEnd', messageId: messageId }));
+
+    db.insert(messages)
+      .values({
+        content: recievedMessage,
+        chatId: chatId,
+        messageId: messageId,
+        role: 'assistant',
+        metadata: JSON.stringify({
+          createdAt: new Date(),
+          ...(sources && sources.length > 0 && { sources }),
+        }),
+      })
+      .execute();
   });
   emitter.on('error', (data) => {
     const parsedData = JSON.parse(data);
@@ -74,8 +103,10 @@ export const handleMessage = async (
   embeddings: Embeddings,
 ) => {
   try {
-    const parsedMessage = JSON.parse(message) as Message;
-    const id = Math.random().toString(36).substring(7);
+    const parsedWSMessage = JSON.parse(message) as WSMessage;
+    const parsedMessage = parsedWSMessage.message;
+
+    const id = crypto.randomBytes(7).toString('hex');
 
     if (!parsedMessage.content)
       return ws.send(
@@ -86,7 +117,7 @@ export const handleMessage = async (
         }),
       );
 
-    const history: BaseMessage[] = parsedMessage.history.map((msg) => {
+    const history: BaseMessage[] = parsedWSMessage.history.map((msg) => {
       if (msg[0] === 'human') {
         return new HumanMessage({
           content: msg[1],
@@ -98,8 +129,9 @@ export const handleMessage = async (
       }
     });
 
-    if (parsedMessage.type === 'message') {
-      const handler = searchHandlers[parsedMessage.focusMode];
+    if (parsedWSMessage.type === 'message') {
+      const handler = searchHandlers[parsedWSMessage.focusMode];
+
       if (handler) {
         const emitter = handler(
           parsedMessage.content,
@@ -107,7 +139,37 @@ export const handleMessage = async (
           llm,
           embeddings,
         );
-        handleEmitterEvents(emitter, ws, id);
+
+        handleEmitterEvents(emitter, ws, id, parsedMessage.chatId);
+
+        const chat = await db.query.chats.findFirst({
+          where: eq(chats.id, parsedMessage.chatId),
+        });
+
+        if (!chat) {
+          await db
+            .insert(chats)
+            .values({
+              id: parsedMessage.chatId,
+              title: parsedMessage.content,
+              createdAt: new Date().toString(),
+              focusMode: parsedWSMessage.focusMode,
+            })
+            .execute();
+        }
+
+        await db
+          .insert(messages)
+          .values({
+            content: parsedMessage.content,
+            chatId: parsedMessage.chatId,
+            messageId: id,
+            role: 'user',
+            metadata: JSON.stringify({
+              createdAt: new Date(),
+            }),
+          })
+          .execute();
       } else {
         ws.send(
           JSON.stringify({

From c51ec8ff0f9f4e82f6438b06a30719547d10e99b Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:09:51 +0530
Subject: [PATCH 10/37] feat(app): implement library feature

---
 ui/app/c/[chatId]/page.tsx          |   7 ++
 ui/app/library/layout.tsx           |  12 +++
 ui/app/library/page.tsx             | 104 +++++++++++++++++++++++
 ui/components/Chat.tsx              |   2 +-
 ui/components/ChatWindow.tsx        | 124 ++++++++++++++++++++++++----
 ui/components/MessageBox.tsx        |   2 +-
 ui/components/MessageBoxLoading.tsx |   2 +-
 ui/components/Sidebar.tsx           |   2 +-
 8 files changed, 234 insertions(+), 21 deletions(-)
 create mode 100644 ui/app/c/[chatId]/page.tsx
 create mode 100644 ui/app/library/layout.tsx
 create mode 100644 ui/app/library/page.tsx

diff --git a/ui/app/c/[chatId]/page.tsx b/ui/app/c/[chatId]/page.tsx
new file mode 100644
index 0000000..dc3c92a
--- /dev/null
+++ b/ui/app/c/[chatId]/page.tsx
@@ -0,0 +1,7 @@
+import ChatWindow from '@/components/ChatWindow';
+
+const Page = ({ params }: { params: { chatId: string } }) => {
+  return <ChatWindow id={params.chatId} />;
+};
+
+export default Page;
diff --git a/ui/app/library/layout.tsx b/ui/app/library/layout.tsx
new file mode 100644
index 0000000..00d4a3b
--- /dev/null
+++ b/ui/app/library/layout.tsx
@@ -0,0 +1,12 @@
+import { Metadata } from 'next';
+import React from 'react';
+
+export const metadata: Metadata = {
+  title: 'Library - Perplexica',
+};
+
+const Layout = ({ children }: { children: React.ReactNode }) => {
+  return <div>{children}</div>;
+};
+
+export default Layout;
diff --git a/ui/app/library/page.tsx b/ui/app/library/page.tsx
new file mode 100644
index 0000000..6ba2fe4
--- /dev/null
+++ b/ui/app/library/page.tsx
@@ -0,0 +1,104 @@
+'use client';
+
+import { formatTimeDifference } from '@/lib/utils';
+import { BookOpenText, ClockIcon, ScanEye } from 'lucide-react';
+import Link from 'next/link';
+import { useEffect, useState } from 'react';
+
+interface Chat {
+  id: string;
+  title: string;
+  createdAt: string;
+  focusMode: string;
+}
+
+const Page = () => {
+  const [chats, setChats] = useState<Chat[]>([]);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    const fetchChats = async () => {
+      setLoading(true);
+
+      const res = await fetch(`${process.env.NEXT_PUBLIC_API_URL}/chats`, {
+        method: 'GET',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      });
+
+      const data = await res.json();
+
+      setChats(data.chats);
+      setLoading(false);
+    };
+
+    fetchChats();
+  }, []);
+
+  return loading ? (
+    <div className="flex flex-row items-center justify-center min-h-screen">
+      <svg
+        aria-hidden="true"
+        className="w-8 h-8 text-light-200 fill-light-secondary dark:text-[#202020] animate-spin dark:fill-[#ffffff3b]"
+        viewBox="0 0 100 101"
+        fill="none"
+        xmlns="http://www.w3.org/2000/svg"
+      >
+        <path
+          d="M100 50.5908C100.003 78.2051 78.1951 100.003 50.5908 100C22.9765 99.9972 0.997224 78.018 1 50.4037C1.00281 22.7993 22.8108 0.997224 50.4251 1C78.0395 1.00281 100.018 22.8108 100 50.4251ZM9.08164 50.594C9.06312 73.3997 27.7909 92.1272 50.5966 92.1457C73.4023 92.1642 92.1298 73.4365 92.1483 50.6308C92.1669 27.8251 73.4392 9.0973 50.6335 9.07878C27.8278 9.06026 9.10003 27.787 9.08164 50.594Z"
+          fill="currentColor"
+        />
+        <path
+          d="M93.9676 39.0409C96.393 38.4037 97.8624 35.9116 96.9801 33.5533C95.1945 28.8227 92.871 24.3692 90.0681 20.348C85.6237 14.1775 79.4473 9.36872 72.0454 6.45794C64.6435 3.54717 56.3134 2.65431 48.3133 3.89319C45.869 4.27179 44.3768 6.77534 45.014 9.20079C45.6512 11.6262 48.1343 13.0956 50.5786 12.717C56.5073 11.8281 62.5542 12.5399 68.0406 14.7911C73.527 17.0422 78.2187 20.7487 81.5841 25.4923C83.7976 28.5886 85.4467 32.059 86.4416 35.7474C87.1273 38.1189 89.5423 39.6781 91.9676 39.0409Z"
+          fill="currentFill"
+        />
+      </svg>
+    </div>
+  ) : (
+    <div>
+      <div className="fixed z-40 top-0 left-0 right-0 lg:pl-[104px] lg:pr-6 lg:px-8 px-4 py-4 lg:py-6 border-b border-light-200 dark:border-dark-200">
+        <div className="flex flex-row items-center space-x-2 max-w-screen-lg lg:mx-auto">
+          <BookOpenText />
+          <h2 className="text-black dark:text-white lg:text-3xl lg:font-medium">
+            Library
+          </h2>
+        </div>
+      </div>
+      {chats.length === 0 && (
+        <div className="flex flex-row items-center justify-center min-h-screen">
+          <p className="text-black/70 dark:text-white/70 text-sm">
+            No chats found.
+          </p>
+        </div>
+      )}
+      {chats.length > 0 && (
+        <div className="flex flex-col pt-16 lg:pt-24">
+          {chats.map((chat, i) => (
+            <div
+              className="flex flex-col space-y-4 border-b border-white-200 dark:border-dark-200 py-6 lg:mx-4"
+              key={i}
+            >
+              <Link
+                href={`/c/${chat.id}`}
+                className="text-black dark:text-white lg:text-xl font-medium truncate transition duration-200 hover:text-[#24A0ED] dark:hover:text-[#24A0ED] cursor-pointer"
+              >
+                {chat.title}
+              </Link>
+              <div className="flex flex-row items-center justify-between w-full">
+                <div className="flex flex-row items-center space-x-1 lg:space-x-1.5 text-black/70 dark:text-white/70">
+                  <ClockIcon size={15} />
+                  <p className="text-xs">
+                    {formatTimeDifference(new Date(), chat.createdAt)} Ago
+                  </p>
+                </div>
+              </div>
+            </div>
+          ))}
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default Page;
diff --git a/ui/components/Chat.tsx b/ui/components/Chat.tsx
index c0dbc92..8c0fb80 100644
--- a/ui/components/Chat.tsx
+++ b/ui/components/Chat.tsx
@@ -53,7 +53,7 @@ const Chat = ({
         const isLast = i === messages.length - 1;
 
         return (
-          <Fragment key={msg.id}>
+          <Fragment key={msg.messageId}>
             <MessageBox
               key={i}
               message={msg}
diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx
index 35ffd1e..2f5cf93 100644
--- a/ui/components/ChatWindow.tsx
+++ b/ui/components/ChatWindow.tsx
@@ -5,12 +5,14 @@ import { Document } from '@langchain/core/documents';
 import Navbar from './Navbar';
 import Chat from './Chat';
 import EmptyChat from './EmptyChat';
+import crypto from 'crypto';
 import { toast } from 'sonner';
 import { useSearchParams } from 'next/navigation';
 import { getSuggestions } from '@/lib/actions';
 
 export type Message = {
-  id: string;
+  messageId: string;
+  chatId: string;
   createdAt: Date;
   content: string;
   role: 'user' | 'assistant';
@@ -20,7 +22,7 @@ export type Message = {
 
 const useSocket = (
   url: string,
-  setIsReady: (ready: boolean) => void,
+  setIsWSReady: (ready: boolean) => void,
   setError: (error: boolean) => void,
 ) => {
   const [ws, setWs] = useState<WebSocket | null>(null);
@@ -120,7 +122,7 @@ const useSocket = (
           console.log('[DEBUG] open');
           clearTimeout(timeoutId);
           setError(false);
-          setIsReady(true);
+          setIsWSReady(true);
         };
 
         ws.onerror = () => {
@@ -145,34 +147,114 @@ const useSocket = (
       ws?.close();
       console.log('[DEBUG] closed');
     };
-  }, [ws, url, setIsReady, setError]);
+  }, [ws, url, setIsWSReady, setError]);
 
   return ws;
 };
 
-const ChatWindow = () => {
+const loadMessages = async (
+  chatId: string,
+  setMessages: (messages: Message[]) => void,
+  setIsMessagesLoaded: (loaded: boolean) => void,
+  setChatHistory: (history: [string, string][]) => void,
+  setFocusMode: (mode: string) => void,
+) => {
+  const res = await fetch(
+    `${process.env.NEXT_PUBLIC_API_URL}/chats/${chatId}`,
+    {
+      method: 'GET',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    },
+  );
+
+  const data = await res.json();
+
+  const messages = data.messages.map((msg: any) => {
+    return {
+      ...msg,
+      ...JSON.parse(msg.metadata),
+    };
+  }) as Message[];
+
+  setMessages(messages);
+
+  const history = messages.map((msg) => {
+    return [msg.role, msg.content];
+  }) as [string, string][];
+
+  console.log('[DEBUG] messages loaded');
+
+  document.title = messages[0].content;
+
+  setChatHistory(history);
+  setFocusMode(data.chat.focusMode);
+  console.log(data);
+  setIsMessagesLoaded(true);
+};
+
+const ChatWindow = ({ id }: { id?: string }) => {
   const searchParams = useSearchParams();
   const initialMessage = searchParams.get('q');
 
-  const [isReady, setIsReady] = useState(false);
+  const [chatId, setChatId] = useState<string | undefined>(id);
+  const [newChatCreated, setNewChatCreated] = useState(false);
+
   const [hasError, setHasError] = useState(false);
+  const [isReady, setIsReady] = useState(false);
+
+  const [isWSReady, setIsWSReady] = useState(false);
   const ws = useSocket(
     process.env.NEXT_PUBLIC_WS_URL!,
-    setIsReady,
+    setIsWSReady,
     setHasError,
   );
 
-  const [chatHistory, setChatHistory] = useState<[string, string][]>([]);
-  const [messages, setMessages] = useState<Message[]>([]);
-  const messagesRef = useRef<Message[]>([]);
   const [loading, setLoading] = useState(false);
   const [messageAppeared, setMessageAppeared] = useState(false);
+
+  const [chatHistory, setChatHistory] = useState<[string, string][]>([]);
+  const [messages, setMessages] = useState<Message[]>([]);
+
   const [focusMode, setFocusMode] = useState('webSearch');
 
+  const [isMessagesLoaded, setIsMessagesLoaded] = useState(false);
+
+  useEffect(() => {
+    if (
+      chatId &&
+      !newChatCreated &&
+      !isMessagesLoaded &&
+      messages.length === 0
+    ) {
+      loadMessages(
+        chatId,
+        setMessages,
+        setIsMessagesLoaded,
+        setChatHistory,
+        setFocusMode,
+      );
+    } else if (!chatId) {
+      setNewChatCreated(true);
+      setIsMessagesLoaded(true);
+      setChatId(crypto.randomBytes(20).toString('hex'));
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const messagesRef = useRef<Message[]>([]);
+
   useEffect(() => {
     messagesRef.current = messages;
   }, [messages]);
 
+  useEffect(() => {
+    if (isMessagesLoaded && isWSReady) {
+      setIsReady(true);
+    }
+  }, [isMessagesLoaded, isWSReady]);
+
   const sendMessage = async (message: string) => {
     if (loading) return;
     setLoading(true);
@@ -182,10 +264,15 @@ const ChatWindow = () => {
     let recievedMessage = '';
     let added = false;
 
+    const messageId = crypto.randomBytes(7).toString('hex');
+
     ws?.send(
       JSON.stringify({
         type: 'message',
-        content: message,
+        message: {
+          chatId: chatId!,
+          content: message,
+        },
         focusMode: focusMode,
         history: [...chatHistory, ['human', message]],
       }),
@@ -195,7 +282,8 @@ const ChatWindow = () => {
       ...prevMessages,
       {
         content: message,
-        id: Math.random().toString(36).substring(7),
+        messageId: messageId,
+        chatId: chatId!,
         role: 'user',
         createdAt: new Date(),
       },
@@ -217,7 +305,8 @@ const ChatWindow = () => {
             ...prevMessages,
             {
               content: '',
-              id: data.messageId,
+              messageId: data.messageId,
+              chatId: chatId!,
               role: 'assistant',
               sources: sources,
               createdAt: new Date(),
@@ -234,7 +323,8 @@ const ChatWindow = () => {
             ...prevMessages,
             {
               content: data.data,
-              id: data.messageId,
+              messageId: data.messageId,
+              chatId: chatId!,
               role: 'assistant',
               sources: sources,
               createdAt: new Date(),
@@ -245,7 +335,7 @@ const ChatWindow = () => {
 
         setMessages((prev) =>
           prev.map((message) => {
-            if (message.id === data.messageId) {
+            if (message.messageId === data.messageId) {
               return { ...message, content: message.content + data.data };
             }
 
@@ -278,7 +368,7 @@ const ChatWindow = () => {
           const suggestions = await getSuggestions(messagesRef.current);
           setMessages((prev) =>
             prev.map((msg) => {
-              if (msg.id === lastMsg.id) {
+              if (msg.messageId === lastMsg.messageId) {
                 return { ...msg, suggestions: suggestions };
               }
               return msg;
@@ -292,7 +382,7 @@ const ChatWindow = () => {
   };
 
   const rewrite = (messageId: string) => {
-    const index = messages.findIndex((msg) => msg.id === messageId);
+    const index = messages.findIndex((msg) => msg.messageId === messageId);
 
     if (index === -1) return;
 
diff --git a/ui/components/MessageBox.tsx b/ui/components/MessageBox.tsx
index 1dce2d0..b111088 100644
--- a/ui/components/MessageBox.tsx
+++ b/ui/components/MessageBox.tsx
@@ -119,7 +119,7 @@ const MessageBox = ({
                     {/*  <button className="p-2 text-black/70 dark:text-white/70 rounded-xl hover:bg-light-secondary dark:hover:bg-dark-secondary transition duration-200 hover:text-black text-black dark:hover:text-white">
                       <Share size={18} />
                     </button> */}
-                    <Rewrite rewrite={rewrite} messageId={message.id} />
+                    <Rewrite rewrite={rewrite} messageId={message.messageId} />
                   </div>
                   <div className="flex flex-row items-center space-x-1">
                     <Copy initialMessage={message.content} message={message} />
diff --git a/ui/components/MessageBoxLoading.tsx b/ui/components/MessageBoxLoading.tsx
index caa6f18..3c53d9e 100644
--- a/ui/components/MessageBoxLoading.tsx
+++ b/ui/components/MessageBoxLoading.tsx
@@ -1,6 +1,6 @@
 const MessageBoxLoading = () => {
   return (
-    <div className="flex flex-col space-y-2 w-full lg:w-9/12 bg-light-primary dark:bg-dark-primary animate-pulse rounded-lg p-3">
+    <div className="flex flex-col space-y-2 w-full lg:w-9/12 bg-light-primary dark:bg-dark-primary animate-pulse rounded-lg py-3">
       <div className="h-2 rounded-full w-full bg-light-secondary dark:bg-dark-secondary" />
       <div className="h-2 rounded-full w-9/12 bg-light-secondary dark:bg-dark-secondary" />
       <div className="h-2 rounded-full w-10/12 bg-light-secondary dark:bg-dark-secondary" />
diff --git a/ui/components/Sidebar.tsx b/ui/components/Sidebar.tsx
index ed8953e..cc2097d 100644
--- a/ui/components/Sidebar.tsx
+++ b/ui/components/Sidebar.tsx
@@ -23,7 +23,7 @@ const Sidebar = ({ children }: { children: React.ReactNode }) => {
     {
       icon: Home,
       href: '/',
-      active: segments.length === 0,
+      active: segments.length === 0 || segments.includes('c'),
       label: 'Home',
     },
     {

From 799f4d6aee478de3b18ab14821119234eb2a9608 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:10:26 +0530
Subject: [PATCH 11/37] feat(docker-compose): implement data volume

---
 .gitignore          | 6 +++++-
 backend.dockerfile  | 5 ++++-
 data/.gitignore     | 2 ++
 docker-compose.yaml | 7 ++++++-
 4 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100644 data/.gitignore

diff --git a/.gitignore b/.gitignore
index d64d5cc..a3dd5cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ yarn-error.log
 # Build output
 /.next/
 /out/
+/dist/
 
 # IDE/Editor specific
 .vscode/
@@ -31,4 +32,7 @@ logs/
 
 # Miscellaneous
 .DS_Store
-Thumbs.db
\ No newline at end of file
+Thumbs.db
+
+# Db
+db.sqlite
\ No newline at end of file
diff --git a/backend.dockerfile b/backend.dockerfile
index 47c5d81..5e482cd 100644
--- a/backend.dockerfile
+++ b/backend.dockerfile
@@ -1,4 +1,4 @@
-FROM node:buster-slim
+FROM nikolaik/python-nodejs:python3.12-nodejs20-bullseye
 
 ARG SEARXNG_API_URL
 
@@ -7,11 +7,14 @@ WORKDIR /home/perplexica
 COPY src /home/perplexica/src
 COPY tsconfig.json /home/perplexica/
 COPY config.toml /home/perplexica/
+COPY drizzle.config.ts /home/perplexica/
 COPY package.json /home/perplexica/
 COPY yarn.lock /home/perplexica/
 
 RUN sed -i "s|SEARXNG = \".*\"|SEARXNG = \"${SEARXNG_API_URL}\"|g" /home/perplexica/config.toml
 
+RUN mkdir /home/perplexica/data
+
 RUN yarn install
 RUN yarn build
 
diff --git a/data/.gitignore b/data/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/data/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 786ecdd..d6f9203 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -19,8 +19,10 @@ services:
       - searxng
     ports:
       - 3001:3001
+    volumes:
+      - backend-dbstore:/home/perplexica/data
     extra_hosts:
-      - "host.docker.internal:host-gateway"
+      - 'host.docker.internal:host-gateway'
     networks:
       - perplexica-network
     restart: unless-stopped
@@ -42,3 +44,6 @@ services:
 
 networks:
   perplexica-network:
+
+volumes:
+  backend-dbstore:

From 7d2344dc8506b7b522931dd7cb151e440398f92d Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:11:10 +0530
Subject: [PATCH 12/37] feat(chats): remove comment

---
 src/routes/chats.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/routes/chats.ts b/src/routes/chats.ts
index f970ba4..aacfb60 100644
--- a/src/routes/chats.ts
+++ b/src/routes/chats.ts
@@ -10,8 +10,6 @@ router.get('/', async (_, res) => {
   try {
     let chats = await db.query.chats.findMany();
 
-    /* Reorder the chats (in opposite direction) */
-
     chats = chats.reverse();
 
     return res.status(200).json({ chats: chats });

From 9588eed71020c1345798a9b2de6d35296e50d4f3 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:17:29 +0530
Subject: [PATCH 13/37] feat(package): bump version

---
 package.json    | 2 +-
 ui/package.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 2f6f7f2..1f6a722 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "perplexica-backend",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "license": "MIT",
   "author": "ItzCrazyKns",
   "scripts": {
diff --git a/ui/package.json b/ui/package.json
index 430ad2d..567ce66 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "perplexica-frontend",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "license": "MIT",
   "author": "ItzCrazyKns",
   "scripts": {

From aeef03fbaf274a61c2322b00e7a11bd8ce38a99f Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:17:43 +0530
Subject: [PATCH 14/37] feat(readme): update todo

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 6e07f54..64d2540 100644
--- a/README.md
+++ b/README.md
@@ -133,11 +133,12 @@ If you wish to use Perplexica as an alternative to traditional search engines li
 
 ## Upcoming Features
 
-- [ ] Finalizing Copilot Mode
 - [x] Add settings page
 - [x] Adding support for local LLMs
-- [ ] Adding Discover and History Saving features
+- [x] History Saving features
 - [x] Introducing various Focus Modes
+- [ ] Finalizing Copilot Mode
+- [ ] Adding Discover
 
 ## Support Us
 

From 022cf55db7eb026f371cd772ac157b24616a12dd Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:38:43 +0530
Subject: [PATCH 15/37] feat(docs): add update docs

---
 docs/installation/UPDATING.md | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 docs/installation/UPDATING.md

diff --git a/docs/installation/UPDATING.md b/docs/installation/UPDATING.md
new file mode 100644
index 0000000..4472b9a
--- /dev/null
+++ b/docs/installation/UPDATING.md
@@ -0,0 +1,33 @@
+# Update Perplexica to the latest version
+
+To update Perplexica to the latest version, follow these steps:
+
+## For Docker users
+
+1. Clone the latest version of Perplexica from GitHub:
+
+```bash
+   git clone https://github.com/ItzCrazyKns/Perplexica.git
+```
+
+2. Navigate to the Project Directory
+
+3. Update and Rebuild Docker Containers:
+4. 
+```bash
+docker compose up -d --build
+```
+5. Once the command completes running go to http://localhost:3000 and verify the latest changes.
+
+## For non Docker users
+
+1. Clone the latest version of Perplexica from GitHub:
+
+```bash
+   git clone https://github.com/ItzCrazyKns/Perplexica.git
+```
+
+2. Navigate to the Project Directory
+3. Execute `npm i` in both the `ui` folder and the root directory.
+4. Once packages are updated, execute `npm run build` in both the `ui` folder and the root directory.
+5. Finally, start both the frontend and the backend by running `npm run start` in both the `ui` folder and the root directory.

From f4b01a29bb9287dc8538f7dfdd79dd93d23df340 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 11:39:23 +0530
Subject: [PATCH 16/37] feat(docs): update docs

---
 docs/installation/UPDATING.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/installation/UPDATING.md b/docs/installation/UPDATING.md
index 4472b9a..fa89b41 100644
--- a/docs/installation/UPDATING.md
+++ b/docs/installation/UPDATING.md
@@ -13,11 +13,11 @@ To update Perplexica to the latest version, follow these steps:
 2. Navigate to the Project Directory
 
 3. Update and Rebuild Docker Containers:
-4. 
+
 ```bash
 docker compose up -d --build
 ```
-5. Once the command completes running go to http://localhost:3000 and verify the latest changes.
+4. Once the command completes running go to http://localhost:3000 and verify the latest changes.
 
 ## For non Docker users
 

From c62684407de848b3d4c01e29be7e57d3bfc4da2d Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 29 Jun 2024 12:11:34 +0530
Subject: [PATCH 17/37] feat(chat-window): handle notFound errors

---
 ui/components/ChatWindow.tsx | 52 +++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx
index 2f5cf93..675df49 100644
--- a/ui/components/ChatWindow.tsx
+++ b/ui/components/ChatWindow.tsx
@@ -9,6 +9,7 @@ import crypto from 'crypto';
 import { toast } from 'sonner';
 import { useSearchParams } from 'next/navigation';
 import { getSuggestions } from '@/lib/actions';
+import Error from 'next/error';
 
 export type Message = {
   messageId: string;
@@ -158,6 +159,7 @@ const loadMessages = async (
   setIsMessagesLoaded: (loaded: boolean) => void,
   setChatHistory: (history: [string, string][]) => void,
   setFocusMode: (mode: string) => void,
+  setNotFound: (notFound: boolean) => void,
 ) => {
   const res = await fetch(
     `${process.env.NEXT_PUBLIC_API_URL}/chats/${chatId}`,
@@ -169,6 +171,12 @@ const loadMessages = async (
     },
   );
 
+  if (res.status === 404) {
+    setNotFound(true);
+    setIsMessagesLoaded(true);
+    return;
+  }
+
   const data = await res.json();
 
   const messages = data.messages.map((msg: any) => {
@@ -190,7 +198,6 @@ const loadMessages = async (
 
   setChatHistory(history);
   setFocusMode(data.chat.focusMode);
-  console.log(data);
   setIsMessagesLoaded(true);
 };
 
@@ -221,6 +228,8 @@ const ChatWindow = ({ id }: { id?: string }) => {
 
   const [isMessagesLoaded, setIsMessagesLoaded] = useState(false);
 
+  const [notFound, setNotFound] = useState(false);
+
   useEffect(() => {
     if (
       chatId &&
@@ -234,6 +243,7 @@ const ChatWindow = ({ id }: { id?: string }) => {
         setIsMessagesLoaded,
         setChatHistory,
         setFocusMode,
+        setNotFound,
       );
     } else if (!chatId) {
       setNewChatCreated(true);
@@ -416,26 +426,30 @@ const ChatWindow = ({ id }: { id?: string }) => {
   }
 
   return isReady ? (
-    <div>
-      {messages.length > 0 ? (
-        <>
-          <Navbar messages={messages} />
-          <Chat
-            loading={loading}
-            messages={messages}
+    notFound ? (
+      <Error statusCode={404} />
+    ) : (
+      <div>
+        {messages.length > 0 ? (
+          <>
+            <Navbar messages={messages} />
+            <Chat
+              loading={loading}
+              messages={messages}
+              sendMessage={sendMessage}
+              messageAppeared={messageAppeared}
+              rewrite={rewrite}
+            />
+          </>
+        ) : (
+          <EmptyChat
             sendMessage={sendMessage}
-            messageAppeared={messageAppeared}
-            rewrite={rewrite}
+            focusMode={focusMode}
+            setFocusMode={setFocusMode}
           />
-        </>
-      ) : (
-        <EmptyChat
-          sendMessage={sendMessage}
-          focusMode={focusMode}
-          setFocusMode={setFocusMode}
-        />
-      )}
-    </div>
+        )}
+      </div>
+    )
   ) : (
     <div className="flex flex-row items-center justify-center min-h-screen">
       <svg

From b29126594485e905f54a6fb40ac5ea3b39a62473 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sun, 30 Jun 2024 10:42:01 +0530
Subject: [PATCH 18/37] feat(package): add `@langchain/community`

---
 package.json |   1 +
 yarn.lock    | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)

diff --git a/package.json b/package.json
index 1f6a722..f141b84 100644
--- a/package.json
+++ b/package.json
@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "@iarna/toml": "^2.2.5",
+    "@langchain/community": "^0.2.16",
     "@langchain/openai": "^0.0.25",
     "@xenova/transformers": "^2.17.1",
     "axios": "^1.6.8",
diff --git a/yarn.lock b/yarn.lock
index d418edf..dceddbd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -307,6 +307,23 @@
     "@jridgewell/resolve-uri" "^3.0.3"
     "@jridgewell/sourcemap-codec" "^1.4.10"
 
+"@langchain/community@^0.2.16":
+  version "0.2.16"
+  resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.2.16.tgz#5888baf7fc7ea272c5f91aaa0e71bc444167262d"
+  integrity sha512-dFDcMabKACvuRd0w6EIRLWf1ubPGZEeEwFt9v1jiEr4HCFxH0OF+iM1QUCcVRbB2fK5lqmKeTD1XAeZV8+AyXA==
+  dependencies:
+    "@langchain/core" "~0.2.11"
+    "@langchain/openai" "~0.1.0"
+    binary-extensions "^2.2.0"
+    expr-eval "^2.0.2"
+    flat "^5.0.2"
+    js-yaml "^4.1.0"
+    langchain "0.2.3"
+    langsmith "~0.1.30"
+    uuid "^9.0.0"
+    zod "^3.22.3"
+    zod-to-json-schema "^3.22.5"
+
 "@langchain/community@~0.0.41":
   version "0.0.43"
   resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.0.43.tgz#017e2f9b3209b3999482f10df5aec2520731a63c"
@@ -320,6 +337,24 @@
     uuid "^9.0.0"
     zod "^3.22.3"
 
+"@langchain/core@>0.1.56 <0.3.0", "@langchain/core@>0.2.0 <0.3.0", "@langchain/core@>=0.2.5 <0.3.0", "@langchain/core@~0.2.0", "@langchain/core@~0.2.11":
+  version "0.2.11"
+  resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.2.11.tgz#5f47467e20e56b250831baef20083657c6facb4c"
+  integrity sha512-d4SNL7WI0c3oHrV4WxCRH1/TNqdePXEzYjYwIb4aEH6lW1aM0utGhLbNthX+aYkOL4Ynx2FoG4h91ECIipiKWQ==
+  dependencies:
+    ansi-styles "^5.0.0"
+    camelcase "6"
+    decamelize "1.2.0"
+    js-tiktoken "^1.0.12"
+    langsmith "~0.1.30"
+    ml-distance "^4.0.0"
+    mustache "^4.2.0"
+    p-queue "^6.6.2"
+    p-retry "4"
+    uuid "^9.0.0"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
 "@langchain/core@~0.1.44", "@langchain/core@~0.1.45":
   version "0.1.52"
   resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.52.tgz#7619310b83ffa841628efe2e1eda873ca714d068"
@@ -348,6 +383,36 @@
     zod "^3.22.4"
     zod-to-json-schema "^3.22.3"
 
+"@langchain/openai@~0.0.28":
+  version "0.0.34"
+  resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.0.34.tgz#36c9bca0721ab9f7e5d40927e7c0429cacbd5b56"
+  integrity sha512-M+CW4oXle5fdoz2T2SwdOef8pl3/1XmUx1vjn2mXUVM/128aO0l23FMF0SNBsAbRV6P+p/TuzjodchJbi0Ht/A==
+  dependencies:
+    "@langchain/core" ">0.1.56 <0.3.0"
+    js-tiktoken "^1.0.12"
+    openai "^4.41.1"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
+"@langchain/openai@~0.1.0":
+  version "0.1.3"
+  resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.1.3.tgz#6eb0994e970d85ffa9aaeafb94449024ccf6ca63"
+  integrity sha512-riv/JC9x2A8b7GcHu8sx+mlZJ8KAwSSi231IPTlcciYnKozmrQ5H0vrtiD31fxiDbaRsk7tyCpkSBIOQEo7CyQ==
+  dependencies:
+    "@langchain/core" ">=0.2.5 <0.3.0"
+    js-tiktoken "^1.0.12"
+    openai "^4.49.1"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
+"@langchain/textsplitters@~0.0.0":
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/@langchain/textsplitters/-/textsplitters-0.0.3.tgz#1a3cc93dd2ab330edb225400ded190a22fea14e3"
+  integrity sha512-cXWgKE3sdWLSqAa8ykbCcUsUF1Kyr5J3HOWYGuobhPEycXW4WI++d5DhzdpL238mzoEXTi90VqfSCra37l5YqA==
+  dependencies:
+    "@langchain/core" ">0.2.0 <0.3.0"
+    js-tiktoken "^1.0.12"
+
 "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2":
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf"
@@ -1508,6 +1573,13 @@ is-stream@^2.0.0:
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077"
   integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==
 
+js-tiktoken@^1.0.12:
+  version "1.0.12"
+  resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.12.tgz#af0f5cf58e5e7318240d050c8413234019424211"
+  integrity sha512-L7wURW1fH9Qaext0VzaUDpFGVQgjkdE3Dgsy9/+yXyGEpBKnylTd0mU0bfbNkKDlXRb6TEsZkwuflu1B8uQbJQ==
+  dependencies:
+    base64-js "^1.5.1"
+
 js-tiktoken@^1.0.7, js-tiktoken@^1.0.8:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.10.tgz#2b343ec169399dcee8f9ef9807dbd4fafd3b30dc"
@@ -1532,6 +1604,28 @@ kuler@^2.0.0:
   resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3"
   integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==
 
+langchain@0.2.3:
+  version "0.2.3"
+  resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.2.3.tgz#c14bb05cf871b21bd63b84b3ab89580b1d62539f"
+  integrity sha512-T9xR7zd+Nj0oXy6WoYKmZLy0DlQiDLFPGYWdOXDxy+AvqlujoPdVQgDSpdqiOHvAjezrByAoKxoHCz5XMwTP/Q==
+  dependencies:
+    "@langchain/core" "~0.2.0"
+    "@langchain/openai" "~0.0.28"
+    "@langchain/textsplitters" "~0.0.0"
+    binary-extensions "^2.2.0"
+    js-tiktoken "^1.0.12"
+    js-yaml "^4.1.0"
+    jsonpointer "^5.0.1"
+    langchainhub "~0.0.8"
+    langsmith "~0.1.7"
+    ml-distance "^4.0.0"
+    openapi-types "^12.1.3"
+    p-retry "4"
+    uuid "^9.0.0"
+    yaml "^2.2.1"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
 langchain@^0.1.30:
   version "0.1.30"
   resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.1.30.tgz#e1adb3f1849fcd5c596c668300afd5dc8cb37a97"
@@ -1571,6 +1665,23 @@ langsmith@~0.1.1, langsmith@~0.1.7:
     p-retry "4"
     uuid "^9.0.0"
 
+langsmith@~0.1.30:
+  version "0.1.34"
+  resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.1.34.tgz#801310495fef258ed9c22bb5575120e2c06d51cf"
+  integrity sha512-aMv2k8kEaovhTuZnK6/6DMCoM7Jurvm1AzdESn+yN+HramRxp3sK32jFRz3ogkXP6GjAjOIofcnNkzhHXSUXGA==
+  dependencies:
+    "@types/uuid" "^9.0.1"
+    commander "^10.0.1"
+    lodash.set "^4.3.2"
+    p-queue "^6.6.2"
+    p-retry "4"
+    uuid "^9.0.0"
+
+lodash.set@^4.3.2:
+  version "4.3.2"
+  resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23"
+  integrity sha512-4hNPN5jlm/N/HLMCO43v8BXKq9Z7QdAGc/VGrRD61w8gN9g/6jF9A4L1pbUgBLCffi0w9VsXfTOij5x8iTyFvg==
+
 logform@^2.3.2, logform@^2.4.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/logform/-/logform-2.6.0.tgz#8c82a983f05d6eaeb2d75e3decae7a768b2bf9b5"
@@ -1714,6 +1825,11 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
+mustache@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/mustache/-/mustache-4.2.0.tgz#e5892324d60a12ec9c2a73359edca52972bf6f64"
+  integrity sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==
+
 napi-build-utils@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806"
@@ -1858,6 +1974,20 @@ openai@^4.26.0:
     node-fetch "^2.6.7"
     web-streams-polyfill "^3.2.1"
 
+openai@^4.41.1, openai@^4.49.1:
+  version "4.52.2"
+  resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.2.tgz#5d67271f3df84c0b54676b08990eaa9402151759"
+  integrity sha512-mMc0XgFuVSkcm0lRIi8zaw++otC82ZlfkCur1qguXYWPETr/+ZwL9A/vvp3YahX+shpaT6j03dwsmUyLAfmEfg==
+  dependencies:
+    "@types/node" "^18.11.18"
+    "@types/node-fetch" "^2.6.4"
+    abort-controller "^3.0.0"
+    agentkeepalive "^4.2.1"
+    form-data-encoder "1.7.2"
+    formdata-node "^4.3.2"
+    node-fetch "^2.6.7"
+    web-streams-polyfill "^3.2.1"
+
 openapi-types@^12.1.3:
   version "12.1.3"
   resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.3.tgz#471995eb26c4b97b7bd356aacf7b91b73e777dd3"
@@ -2462,6 +2592,11 @@ zod-to-json-schema@^3.22.3:
   resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.22.5.tgz#3646e81cfc318dbad2a22519e5ce661615418673"
   integrity sha512-+akaPo6a0zpVCCseDed504KBJUQpEW5QZw7RMneNmKw+fGaML1Z9tUNLnHHAC8x6dzVRO1eB2oEMyZRnuBZg7Q==
 
+zod-to-json-schema@^3.22.5:
+  version "3.23.1"
+  resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.1.tgz#5225925b8ed5fa20096bd99be076c4b29b53d309"
+  integrity sha512-oT9INvydob1XV0v1d2IadrR74rLtDInLvDFfAa1CG0Pmg/vxATk7I2gSelfj271mbzeM4Da0uuDQE/Nkj3DWNw==
+
 zod@^3.22.3, zod@^3.22.4:
   version "3.22.4"
   resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.4.tgz#f31c3a9386f61b1f228af56faa9255e845cf3fff"

From 6ae825999a953bf1dc66c6b5dd8c81b4b7741362 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sun, 30 Jun 2024 10:45:35 +0530
Subject: [PATCH 19/37] feat(readme): update manual install

---
 README.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 64d2540..da14052 100644
--- a/README.md
+++ b/README.md
@@ -85,11 +85,12 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker.
 
 ### Non-Docker Installation
 
-1. Clone the repository and rename the `sample.config.toml` file to `config.toml` in the root directory. Ensure you complete all required fields in this file.
-2. Rename the `.env.example` file to `.env` in the `ui` folder and fill in all necessary fields.
-3. After populating the configuration and environment files, run `npm i` in both the `ui` folder and the root directory.
-4. Install the dependencies and then execute `npm run build` in both the `ui` folder and the root directory.
-5. Finally, start both the frontend and the backend by running `npm run start` in both the `ui` folder and the root directory.
+1. Install SearXNG and allow `JSON` format in the SearXNG settings.
+2. Clone the repository and rename the `sample.config.toml` file to `config.toml` in the root directory. Ensure you complete all required fields in this file.
+3. Rename the `.env.example` file to `.env` in the `ui` folder and fill in all necessary fields.
+4. After populating the configuration and environment files, run `npm i` in both the `ui` folder and the root directory.
+5. Install the dependencies and then execute `npm run build` in both the `ui` folder and the root directory.
+6. Finally, start both the frontend and the backend by running `npm run start` in both the `ui` folder and the root directory.
 
 **Note**: Using Docker is recommended as it simplifies the setup process, especially for managing environment variables and dependencies.
 

From fcf9b644af65218c68a74cd55df1834f5d3b6973 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <95534749+ItzCrazyKns@users.noreply.github.com>
Date: Sun, 30 Jun 2024 12:34:32 +0530
Subject: [PATCH 20/37] Create FUNDING.yml

---
 .github/FUNDING.yml | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .github/FUNDING.yml

diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
new file mode 100644
index 0000000..faa9fa9
--- /dev/null
+++ b/.github/FUNDING.yml
@@ -0,0 +1 @@
+patreon: itzcrazykns

From ce593daab95c2213ecd4e63047cee9f7ecf769d5 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <95534749+ItzCrazyKns@users.noreply.github.com>
Date: Sun, 30 Jun 2024 12:39:37 +0530
Subject: [PATCH 21/37] Update README.md

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index da14052..acd11f5 100644
--- a/README.md
+++ b/README.md
@@ -147,11 +147,11 @@ If you find Perplexica useful, consider giving us a star on GitHub. This helps m
 
 ### Donations
 
-We also accept donations to help sustain our project. If you would like to contribute, you can use the following button to make a donation in cryptocurrency. Thank you for your support!
+We also accept donations to help sustain our project. If you would like to contribute, you can use the following options to donate. Thank you for your support!
 
-<a href="https://nowpayments.io/donation?api_key=RFFKJH1-GRR4DQG-HFV1DZP-00G6MMK&source=lk_donation&medium=referral" target="_blank">
-  <img src="https://nowpayments.io/images/embeds/donation-button-white.svg" alt="Crypto donation button by NOWPayments">
-</a>
+| Cards  | Ethereum |
+|---|---|
+|  https://www.patreon.com/itzcrazykns  | Address: `0xB025a84b2F269570Eb8D4b05DEdaA41D8525B6DD`  |
 
 ## Contribution
 

From c74e16e01cbc820f275e59094eea4dd4df90a026 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Tue, 2 Jul 2024 10:51:47 +0530
Subject: [PATCH 22/37] feat(chats): add delete functionality

---
 src/routes/chats.ts          |  23 +++++++
 ui/app/library/page.tsx      |  10 ++-
 ui/components/DeleteChat.tsx | 114 +++++++++++++++++++++++++++++++++++
 3 files changed, 145 insertions(+), 2 deletions(-)
 create mode 100644 ui/components/DeleteChat.tsx

diff --git a/src/routes/chats.ts b/src/routes/chats.ts
index aacfb60..afa74f9 100644
--- a/src/routes/chats.ts
+++ b/src/routes/chats.ts
@@ -40,4 +40,27 @@ router.get('/:id', async (req, res) => {
   }
 });
 
+router.delete(`/:id`, async (req, res) => {
+  try {
+    const chatExists = await db.query.chats.findFirst({
+      where: eq(chats.id, req.params.id),
+    });
+
+    if (!chatExists) {
+      return res.status(404).json({ message: 'Chat not found' });
+    }
+
+    await db.delete(chats).where(eq(chats.id, req.params.id)).execute();
+    await db
+      .delete(messages)
+      .where(eq(messages.chatId, req.params.id))
+      .execute();
+
+    return res.status(200).json({ message: 'Chat deleted successfully' });
+  } catch (err) {
+    res.status(500).json({ message: 'An error has occurred.' });
+    logger.error(`Error in deleting chat: ${err.message}`);
+  }
+});
+
 export default router;
diff --git a/ui/app/library/page.tsx b/ui/app/library/page.tsx
index 6ba2fe4..8294fc1 100644
--- a/ui/app/library/page.tsx
+++ b/ui/app/library/page.tsx
@@ -1,11 +1,12 @@
 'use client';
 
+import DeleteChat from '@/components/DeleteChat';
 import { formatTimeDifference } from '@/lib/utils';
-import { BookOpenText, ClockIcon, ScanEye } from 'lucide-react';
+import { BookOpenText, ClockIcon, Delete, ScanEye } from 'lucide-react';
 import Link from 'next/link';
 import { useEffect, useState } from 'react';
 
-interface Chat {
+export interface Chat {
   id: string;
   title: string;
   createdAt: string;
@@ -92,6 +93,11 @@ const Page = () => {
                     {formatTimeDifference(new Date(), chat.createdAt)} Ago
                   </p>
                 </div>
+                <DeleteChat
+                  chatId={chat.id}
+                  chats={chats}
+                  setChats={setChats}
+                />
               </div>
             </div>
           ))}
diff --git a/ui/components/DeleteChat.tsx b/ui/components/DeleteChat.tsx
new file mode 100644
index 0000000..165f86e
--- /dev/null
+++ b/ui/components/DeleteChat.tsx
@@ -0,0 +1,114 @@
+import { Delete, Trash } from 'lucide-react';
+import { Dialog, Transition } from '@headlessui/react';
+import { Fragment, useState } from 'react';
+import { toast } from 'sonner';
+import { Chat } from '@/app/library/page';
+
+const DeleteChat = ({
+  chatId,
+  chats,
+  setChats,
+}: {
+  chatId: string;
+  chats: Chat[];
+  setChats: (chats: Chat[]) => void;
+}) => {
+  const [confirmationDialogOpen, setConfirmationDialogOpen] = useState(false);
+  const [loading, setLoading] = useState(false);
+
+  const handleDelete = async () => {
+    setLoading(true);
+    try {
+      const res = await fetch(
+        `${process.env.NEXT_PUBLIC_API_URL}/chats/${chatId}`,
+        {
+          method: 'DELETE',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+        },
+      );
+
+      if (res.status != 200) {
+        throw new Error('Failed to delete chat');
+      }
+
+      const newChats = chats.filter((chat) => chat.id !== chatId);
+
+      setChats(newChats);
+    } catch (err: any) {
+      toast.error(err.message);
+    } finally {
+      setConfirmationDialogOpen(false);
+      setLoading(false);
+    }
+  };
+
+  return (
+    <>
+      <button
+        onClick={() => {
+          setConfirmationDialogOpen(true);
+        }}
+        className="bg-transparent text-red-400 hover:scale-105 transition duration-200"
+      >
+        <Trash size={17} />
+      </button>
+      <Transition appear show={confirmationDialogOpen} as={Fragment}>
+        <Dialog
+          as="div"
+          className="relative z-50"
+          onClose={() => {
+            if (!loading) {
+              setConfirmationDialogOpen(false);
+            }
+          }}
+        >
+          <Dialog.Backdrop className="fixed inset-0 bg-black/30" />
+          <div className="fixed inset-0 overflow-y-auto">
+            <div className="flex min-h-full items-center justify-center p-4 text-center">
+              <Transition.Child
+                as={Fragment}
+                enter="ease-out duration-200"
+                enterFrom="opacity-0 scale-95"
+                enterTo="opacity-100 scale-100"
+                leave="ease-in duration-100"
+                leaveFrom="opacity-100 scale-200"
+                leaveTo="opacity-0 scale-95"
+              >
+                <Dialog.Panel className="w-full max-w-md transform rounded-2xl bg-light-secondary dark:bg-dark-secondary border border-light-200 dark:border-dark-200 p-6 text-left align-middle shadow-xl transition-all">
+                  <Dialog.Title className="text-lg font-medium leading-6 dark:text-white">
+                    Delete Confirmation
+                  </Dialog.Title>
+                  <Dialog.Description className="text-sm dark:text-white/70 text-black/70">
+                    Are you sure you want to delete this chat?
+                  </Dialog.Description>
+                  <div className="flex flex-row items-end justify-end space-x-4 mt-6">
+                    <button
+                      onClick={() => {
+                        if (!loading) {
+                          setConfirmationDialogOpen(false);
+                        }
+                      }}
+                      className="text-black/50 dark:text-white/50 text-sm hover:text-black/70 hover:dark:text-white/70 transition duration-200"
+                    >
+                      Cancel
+                    </button>
+                    <button
+                      onClick={handleDelete}
+                      className="text-red-400 text-sm hover:text-red-500 transition duration200"
+                    >
+                      Delete
+                    </button>
+                  </div>
+                </Dialog.Panel>
+              </Transition.Child>
+            </div>
+          </div>
+        </Dialog>
+      </Transition>
+    </>
+  );
+};
+
+export default DeleteChat;

From 5217d21366ca41f5a4693e43f36b7bb79083359e Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Wed, 3 Jul 2024 20:47:31 +0530
Subject: [PATCH 23/37] feat(dockerfile): revert to `node:slim`

---
 backend.dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend.dockerfile b/backend.dockerfile
index 5e482cd..4886573 100644
--- a/backend.dockerfile
+++ b/backend.dockerfile
@@ -1,4 +1,4 @@
-FROM nikolaik/python-nodejs:python3.12-nodejs20-bullseye
+FROM node:slim
 
 ARG SEARXNG_API_URL
 
@@ -15,7 +15,7 @@ RUN sed -i "s|SEARXNG = \".*\"|SEARXNG = \"${SEARXNG_API_URL}\"|g" /home/perplex
 
 RUN mkdir /home/perplexica/data
 
-RUN yarn install
+RUN yarn install 
 RUN yarn build
 
 CMD ["yarn", "start"]
\ No newline at end of file

From 80818983d8af50bfb9c7c3e43c8c073297e54439 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Wed, 3 Jul 2024 20:49:13 +0530
Subject: [PATCH 24/37] feat(package): bump version

---
 package.json    | 2 +-
 ui/package.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index f141b84..4f2bb32 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "perplexica-backend",
-  "version": "1.7.0",
+  "version": "1.7.1",
   "license": "MIT",
   "author": "ItzCrazyKns",
   "scripts": {
diff --git a/ui/package.json b/ui/package.json
index 567ce66..bec350a 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "perplexica-frontend",
-  "version": "1.7.0",
+  "version": "1.7.1",
   "license": "MIT",
   "author": "ItzCrazyKns",
   "scripts": {

From c63c9b5c8ad9f587e6d7fcbaf025d2a1bd69e8a2 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Wed, 3 Jul 2024 21:02:21 +0530
Subject: [PATCH 25/37] feat(readme): update ollama guide

---
 README.md | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/README.md b/README.md
index acd11f5..9e7f7d8 100644
--- a/README.md
+++ b/README.md
@@ -111,11 +111,7 @@ If you're encountering an Ollama connection error, it is likely due to the backe
 
 3. **Linux Users - Expose Ollama to Network:**
 
-   - Serve Ollama over your network with the command:
-
-     ```bash
-     OLLAMA_HOST=0.0.0.0 ollama serve
-     ```
+   - Inside `/etc/systemd/system/ollama.service`, you need to add `Environment="OLLAMA_HOST=0.0.0.0"`. Then restart Ollama by `systemctl restart ollama`. For more information see [Ollama docs](https://github.com/ollama/ollama/blob/main/docs/faq.md#setting-environment-variables-on-linux)
 
    - Ensure that the port (default is 11434) is not blocked by your firewall.
 

From 25b5dbd63e9733766c2e0a90e3ac13522b43d4a7 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 6 Jul 2024 14:19:33 +0530
Subject: [PATCH 26/37] feat(providers): separate each provider

---
 README.md                         |   6 +-
 backend.dockerfile                |   2 +-
 src/lib/providers.ts              | 187 ------------------------------
 src/lib/providers/groq.ts         |  57 +++++++++
 src/lib/providers/index.ts        |  36 ++++++
 src/lib/providers/ollama.ts       |  59 ++++++++++
 src/lib/providers/openai.ts       |  59 ++++++++++
 src/lib/providers/transformers.ts |  23 ++++
 8 files changed, 238 insertions(+), 191 deletions(-)
 delete mode 100644 src/lib/providers.ts
 create mode 100644 src/lib/providers/groq.ts
 create mode 100644 src/lib/providers/index.ts
 create mode 100644 src/lib/providers/ollama.ts
 create mode 100644 src/lib/providers/openai.ts
 create mode 100644 src/lib/providers/transformers.ts

diff --git a/README.md b/README.md
index 9e7f7d8..d1388b0 100644
--- a/README.md
+++ b/README.md
@@ -145,9 +145,9 @@ If you find Perplexica useful, consider giving us a star on GitHub. This helps m
 
 We also accept donations to help sustain our project. If you would like to contribute, you can use the following options to donate. Thank you for your support!
 
-| Cards  | Ethereum |
-|---|---|
-|  https://www.patreon.com/itzcrazykns  | Address: `0xB025a84b2F269570Eb8D4b05DEdaA41D8525B6DD`  |
+| Cards                               | Ethereum                                              |
+| ----------------------------------- | ----------------------------------------------------- |
+| https://www.patreon.com/itzcrazykns | Address: `0xB025a84b2F269570Eb8D4b05DEdaA41D8525B6DD` |
 
 ## Contribution
 
diff --git a/backend.dockerfile b/backend.dockerfile
index 4886573..910aae7 100644
--- a/backend.dockerfile
+++ b/backend.dockerfile
@@ -1,4 +1,4 @@
-FROM node:slim
+FROM node:buster-slim
 
 ARG SEARXNG_API_URL
 
diff --git a/src/lib/providers.ts b/src/lib/providers.ts
deleted file mode 100644
index 3223193..0000000
--- a/src/lib/providers.ts
+++ /dev/null
@@ -1,187 +0,0 @@
-import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai';
-import { ChatOllama } from '@langchain/community/chat_models/ollama';
-import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama';
-import { HuggingFaceTransformersEmbeddings } from './huggingfaceTransformer';
-import {
-  getGroqApiKey,
-  getOllamaApiEndpoint,
-  getOpenaiApiKey,
-} from '../config';
-import logger from '../utils/logger';
-
-export const getAvailableChatModelProviders = async () => {
-  const openAIApiKey = getOpenaiApiKey();
-  const groqApiKey = getGroqApiKey();
-  const ollamaEndpoint = getOllamaApiEndpoint();
-
-  const models = {};
-
-  if (openAIApiKey) {
-    try {
-      models['openai'] = {
-        'GPT-3.5 turbo': new ChatOpenAI({
-          openAIApiKey,
-          modelName: 'gpt-3.5-turbo',
-          temperature: 0.7,
-        }),
-        'GPT-4': new ChatOpenAI({
-          openAIApiKey,
-          modelName: 'gpt-4',
-          temperature: 0.7,
-        }),
-        'GPT-4 turbo': new ChatOpenAI({
-          openAIApiKey,
-          modelName: 'gpt-4-turbo',
-          temperature: 0.7,
-        }),
-        'GPT-4 omni': new ChatOpenAI({
-          openAIApiKey,
-          modelName: 'gpt-4o',
-          temperature: 0.7,
-        }),
-      };
-    } catch (err) {
-      logger.error(`Error loading OpenAI models: ${err}`);
-    }
-  }
-
-  if (groqApiKey) {
-    try {
-      models['groq'] = {
-        'LLaMA3 8b': new ChatOpenAI(
-          {
-            openAIApiKey: groqApiKey,
-            modelName: 'llama3-8b-8192',
-            temperature: 0.7,
-          },
-          {
-            baseURL: 'https://api.groq.com/openai/v1',
-          },
-        ),
-        'LLaMA3 70b': new ChatOpenAI(
-          {
-            openAIApiKey: groqApiKey,
-            modelName: 'llama3-70b-8192',
-            temperature: 0.7,
-          },
-          {
-            baseURL: 'https://api.groq.com/openai/v1',
-          },
-        ),
-        'Mixtral 8x7b': new ChatOpenAI(
-          {
-            openAIApiKey: groqApiKey,
-            modelName: 'mixtral-8x7b-32768',
-            temperature: 0.7,
-          },
-          {
-            baseURL: 'https://api.groq.com/openai/v1',
-          },
-        ),
-        'Gemma 7b': new ChatOpenAI(
-          {
-            openAIApiKey: groqApiKey,
-            modelName: 'gemma-7b-it',
-            temperature: 0.7,
-          },
-          {
-            baseURL: 'https://api.groq.com/openai/v1',
-          },
-        ),
-      };
-    } catch (err) {
-      logger.error(`Error loading Groq models: ${err}`);
-    }
-  }
-
-  if (ollamaEndpoint) {
-    try {
-      const response = await fetch(`${ollamaEndpoint}/api/tags`, {
-        headers: {
-          'Content-Type': 'application/json',
-        },
-      });
-
-      const { models: ollamaModels } = (await response.json()) as any;
-
-      models['ollama'] = ollamaModels.reduce((acc, model) => {
-        acc[model.model] = new ChatOllama({
-          baseUrl: ollamaEndpoint,
-          model: model.model,
-          temperature: 0.7,
-        });
-        return acc;
-      }, {});
-    } catch (err) {
-      logger.error(`Error loading Ollama models: ${err}`);
-    }
-  }
-
-  models['custom_openai'] = {};
-
-  return models;
-};
-
-export const getAvailableEmbeddingModelProviders = async () => {
-  const openAIApiKey = getOpenaiApiKey();
-  const ollamaEndpoint = getOllamaApiEndpoint();
-
-  const models = {};
-
-  if (openAIApiKey) {
-    try {
-      models['openai'] = {
-        'Text embedding 3 small': new OpenAIEmbeddings({
-          openAIApiKey,
-          modelName: 'text-embedding-3-small',
-        }),
-        'Text embedding 3 large': new OpenAIEmbeddings({
-          openAIApiKey,
-          modelName: 'text-embedding-3-large',
-        }),
-      };
-    } catch (err) {
-      logger.error(`Error loading OpenAI embeddings: ${err}`);
-    }
-  }
-
-  if (ollamaEndpoint) {
-    try {
-      const response = await fetch(`${ollamaEndpoint}/api/tags`, {
-        headers: {
-          'Content-Type': 'application/json',
-        },
-      });
-
-      const { models: ollamaModels } = (await response.json()) as any;
-
-      models['ollama'] = ollamaModels.reduce((acc, model) => {
-        acc[model.model] = new OllamaEmbeddings({
-          baseUrl: ollamaEndpoint,
-          model: model.model,
-        });
-        return acc;
-      }, {});
-    } catch (err) {
-      logger.error(`Error loading Ollama embeddings: ${err}`);
-    }
-  }
-
-  try {
-    models['local'] = {
-      'BGE Small': new HuggingFaceTransformersEmbeddings({
-        modelName: 'Xenova/bge-small-en-v1.5',
-      }),
-      'GTE Small': new HuggingFaceTransformersEmbeddings({
-        modelName: 'Xenova/gte-small',
-      }),
-      'Bert Multilingual': new HuggingFaceTransformersEmbeddings({
-        modelName: 'Xenova/bert-base-multilingual-uncased',
-      }),
-    };
-  } catch (err) {
-    logger.error(`Error loading local embeddings: ${err}`);
-  }
-
-  return models;
-};
diff --git a/src/lib/providers/groq.ts b/src/lib/providers/groq.ts
new file mode 100644
index 0000000..ecdce4d
--- /dev/null
+++ b/src/lib/providers/groq.ts
@@ -0,0 +1,57 @@
+import { ChatOpenAI } from '@langchain/openai';
+import { getGroqApiKey } from '../../config';
+import logger from '../../utils/logger';
+
+export const loadGroqChatModels = async () => {
+  const groqApiKey = getGroqApiKey();
+
+  try {
+    const chatModels = {
+      'LLaMA3 8b': new ChatOpenAI(
+        {
+          openAIApiKey: groqApiKey,
+          modelName: 'llama3-8b-8192',
+          temperature: 0.7,
+        },
+        {
+          baseURL: 'https://api.groq.com/openai/v1',
+        },
+      ),
+      'LLaMA3 70b': new ChatOpenAI(
+        {
+          openAIApiKey: groqApiKey,
+          modelName: 'llama3-70b-8192',
+          temperature: 0.7,
+        },
+        {
+          baseURL: 'https://api.groq.com/openai/v1',
+        },
+      ),
+      'Mixtral 8x7b': new ChatOpenAI(
+        {
+          openAIApiKey: groqApiKey,
+          modelName: 'mixtral-8x7b-32768',
+          temperature: 0.7,
+        },
+        {
+          baseURL: 'https://api.groq.com/openai/v1',
+        },
+      ),
+      'Gemma 7b': new ChatOpenAI(
+        {
+          openAIApiKey: groqApiKey,
+          modelName: 'gemma-7b-it',
+          temperature: 0.7,
+        },
+        {
+          baseURL: 'https://api.groq.com/openai/v1',
+        },
+      ),
+    };
+
+    return chatModels;
+  } catch (err) {
+    logger.error(`Error loading Groq models: ${err}`);
+    return {};
+  }
+};
diff --git a/src/lib/providers/index.ts b/src/lib/providers/index.ts
new file mode 100644
index 0000000..5807f94
--- /dev/null
+++ b/src/lib/providers/index.ts
@@ -0,0 +1,36 @@
+import { loadGroqChatModels } from './groq';
+import { loadOllamaChatModels } from './ollama';
+import { loadOpenAIChatModels, loadOpenAIEmbeddingsModel } from './openai';
+import { loadTransformersEmbeddingsModel } from './transformers';
+
+const chatModelProviders = {
+  openai: loadOpenAIChatModels,
+  groq: loadGroqChatModels,
+  ollama: loadOllamaChatModels,
+};
+
+const embeddingModelProviders = {
+  openai: loadOpenAIEmbeddingsModel,
+  local: loadTransformersEmbeddingsModel,
+  ollama: loadOllamaChatModels,
+};
+
+export const getAvailableChatModelProviders = async () => {
+  const models = {};
+
+  for (const provider in chatModelProviders) {
+    models[provider] = await chatModelProviders[provider]();
+  }
+
+  return models;
+};
+
+export const getAvailableEmbeddingModelProviders = async () => {
+  const models = {};
+
+  for (const provider in embeddingModelProviders) {
+    models[provider] = await embeddingModelProviders[provider]();
+  }
+
+  return models;
+};
diff --git a/src/lib/providers/ollama.ts b/src/lib/providers/ollama.ts
new file mode 100644
index 0000000..febe5e8
--- /dev/null
+++ b/src/lib/providers/ollama.ts
@@ -0,0 +1,59 @@
+import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama';
+import { getOllamaApiEndpoint } from '../../config';
+import logger from '../../utils/logger';
+import { ChatOllama } from '@langchain/community/chat_models/ollama';
+
+export const loadOllamaChatModels = async () => {
+  const ollamaEndpoint = getOllamaApiEndpoint();
+
+  try {
+    const response = await fetch(`${ollamaEndpoint}/api/tags`, {
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    const { models: ollamaModels } = (await response.json()) as any;
+
+    const chatModels = ollamaModels.reduce((acc, model) => {
+      acc[model.model] = new ChatOllama({
+        baseUrl: ollamaEndpoint,
+        model: model.model,
+        temperature: 0.7,
+      });
+      return acc;
+    }, {});
+
+    return chatModels;
+  } catch (err) {
+    logger.error(`Error loading Ollama models: ${err}`);
+    return {};
+  }
+};
+
+export const loadOpenAIEmbeddingsModel = async () => {
+  const ollamaEndpoint = getOllamaApiEndpoint();
+
+  try {
+    const response = await fetch(`${ollamaEndpoint}/api/tags`, {
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    const { models: ollamaModels } = (await response.json()) as any;
+
+    const embeddingsModels = ollamaModels.reduce((acc, model) => {
+      acc[model.model] = new OllamaEmbeddings({
+        baseUrl: ollamaEndpoint,
+        model: model.model,
+      });
+      return acc;
+    }, {});
+
+    return embeddingsModels;
+  } catch (err) {
+    logger.error(`Error loading Ollama embeddings model: ${err}`);
+    return {};
+  }
+};
diff --git a/src/lib/providers/openai.ts b/src/lib/providers/openai.ts
new file mode 100644
index 0000000..705f1a4
--- /dev/null
+++ b/src/lib/providers/openai.ts
@@ -0,0 +1,59 @@
+import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai';
+import { getOpenaiApiKey } from '../../config';
+import logger from '../../utils/logger';
+
+export const loadOpenAIChatModels = async () => {
+  const openAIApiKey = getOpenaiApiKey();
+
+  try {
+    const chatModels = {
+      'GPT-3.5 turbo': new ChatOpenAI({
+        openAIApiKey,
+        modelName: 'gpt-3.5-turbo',
+        temperature: 0.7,
+      }),
+      'GPT-4': new ChatOpenAI({
+        openAIApiKey,
+        modelName: 'gpt-4',
+        temperature: 0.7,
+      }),
+      'GPT-4 turbo': new ChatOpenAI({
+        openAIApiKey,
+        modelName: 'gpt-4-turbo',
+        temperature: 0.7,
+      }),
+      'GPT-4 omni': new ChatOpenAI({
+        openAIApiKey,
+        modelName: 'gpt-4o',
+        temperature: 0.7,
+      }),
+    };
+
+    return chatModels;
+  } catch (err) {
+    logger.error(`Error loading OpenAI models: ${err}`);
+    return {};
+  }
+};
+
+export const loadOpenAIEmbeddingsModel = async () => {
+  const openAIApiKey = getOpenaiApiKey();
+
+  try {
+    const embeddingModels = {
+      'Text embedding 3 small': new OpenAIEmbeddings({
+        openAIApiKey,
+        modelName: 'text-embedding-3-small',
+      }),
+      'Text embedding 3 large': new OpenAIEmbeddings({
+        openAIApiKey,
+        modelName: 'text-embedding-3-large',
+      }),
+    };
+
+    return embeddingModels;
+  } catch (err) {
+    logger.error(`Error loading OpenAI embeddings model: ${err}`);
+    return {};
+  }
+};
diff --git a/src/lib/providers/transformers.ts b/src/lib/providers/transformers.ts
new file mode 100644
index 0000000..7ef8596
--- /dev/null
+++ b/src/lib/providers/transformers.ts
@@ -0,0 +1,23 @@
+import logger from '../../utils/logger';
+import { HuggingFaceTransformersEmbeddings } from '../huggingfaceTransformer';
+
+export const loadTransformersEmbeddingsModel = async () => {
+  try {
+    const embeddingModels = {
+      'BGE Small': new HuggingFaceTransformersEmbeddings({
+        modelName: 'Xenova/bge-small-en-v1.5',
+      }),
+      'GTE Small': new HuggingFaceTransformersEmbeddings({
+        modelName: 'Xenova/gte-small',
+      }),
+      'Bert Multilingual': new HuggingFaceTransformersEmbeddings({
+        modelName: 'Xenova/bert-base-multilingual-uncased',
+      }),
+    };
+
+    return embeddingModels;
+  } catch (err) {
+    logger.error(`Error loading Transformers embeddings model: ${err}`);
+    return {};
+  }
+};

From 2678c36e448b966e0e18d89e16ddfaa06caf0239 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 6 Jul 2024 15:12:51 +0530
Subject: [PATCH 27/37] feat(agents): fix grammar in prompt, closes 239 & 203

---
 src/agents/academicSearchAgent.ts     | 4 ++--
 src/agents/redditSearchAgent.ts       | 4 ++--
 src/agents/webSearchAgent.ts          | 4 ++--
 src/agents/wolframAlphaSearchAgent.ts | 4 ++--
 src/agents/youtubeSearchAgent.ts      | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/agents/academicSearchAgent.ts b/src/agents/academicSearchAgent.ts
index 5c11307..d797119 100644
--- a/src/agents/academicSearchAgent.ts
+++ b/src/agents/academicSearchAgent.ts
@@ -44,7 +44,7 @@ Rephrased question:
 const basicAcademicSearchResponsePrompt = `
     You are Perplexica, an AI model who is expert at searching the web and answering user's queries. You are set on focus mode 'Academic', this means you will be searching for academic papers and articles on the web.
 
-    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containg a brief description of the content of that page).
+    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containing a brief description of the content of that page).
     You must use this context to answer the user's query in the best way possible. Use an unbaised and journalistic tone in your response. Do not repeat the text.
     You must not tell the user to open any link or visit any website to get the answer. You must provide the answer in the response itself. If the user asks for links you can provide them.
     Your responses should be medium to long in length be informative and relevant to the user's query. You can use markdowns to format your response. You should use bullet points to list the information. Make sure the answer is not short and is informative.
@@ -52,7 +52,7 @@ const basicAcademicSearchResponsePrompt = `
     Place these citations at the end of that particular sentence. You can cite the same sentence multiple times if it is relevant to the user's query like [number1][number2].
     However you do not need to cite it using the same number. You can use different numbers to cite the same sentence multiple times. The number refers to the number of the search result (passed in the context) used to generate that part of the answer.
 
-    Aything inside the following \`context\` HTML block provided below is for your knowledge returned by the search engine and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
+    Anything inside the following \`context\` HTML block provided below is for your knowledge returned by the search engine and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
     talk about the context in your response. 
 
     <context>
diff --git a/src/agents/redditSearchAgent.ts b/src/agents/redditSearchAgent.ts
index 34e9ec2..3c60c68 100644
--- a/src/agents/redditSearchAgent.ts
+++ b/src/agents/redditSearchAgent.ts
@@ -44,7 +44,7 @@ Rephrased question:
 const basicRedditSearchResponsePrompt = `
     You are Perplexica, an AI model who is expert at searching the web and answering user's queries. You are set on focus mode 'Reddit', this means you will be searching for information, opinions and discussions on the web using Reddit.
 
-    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containg a brief description of the content of that page).
+    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containing a brief description of the content of that page).
     You must use this context to answer the user's query in the best way possible. Use an unbaised and journalistic tone in your response. Do not repeat the text.
     You must not tell the user to open any link or visit any website to get the answer. You must provide the answer in the response itself. If the user asks for links you can provide them.
     Your responses should be medium to long in length be informative and relevant to the user's query. You can use markdowns to format your response. You should use bullet points to list the information. Make sure the answer is not short and is informative.
@@ -52,7 +52,7 @@ const basicRedditSearchResponsePrompt = `
     Place these citations at the end of that particular sentence. You can cite the same sentence multiple times if it is relevant to the user's query like [number1][number2].
     However you do not need to cite it using the same number. You can use different numbers to cite the same sentence multiple times. The number refers to the number of the search result (passed in the context) used to generate that part of the answer.
 
-    Aything inside the following \`context\` HTML block provided below is for your knowledge returned by Reddit and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
+    Anything inside the following \`context\` HTML block provided below is for your knowledge returned by Reddit and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
     talk about the context in your response. 
 
     <context>
diff --git a/src/agents/webSearchAgent.ts b/src/agents/webSearchAgent.ts
index 1364742..04de148 100644
--- a/src/agents/webSearchAgent.ts
+++ b/src/agents/webSearchAgent.ts
@@ -44,7 +44,7 @@ Rephrased question:
 const basicWebSearchResponsePrompt = `
     You are Perplexica, an AI model who is expert at searching the web and answering user's queries.
 
-    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containg a brief description of the content of that page).
+    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containing a brief description of the content of that page).
     You must use this context to answer the user's query in the best way possible. Use an unbaised and journalistic tone in your response. Do not repeat the text.
     You must not tell the user to open any link or visit any website to get the answer. You must provide the answer in the response itself. If the user asks for links you can provide them.
     Your responses should be medium to long in length be informative and relevant to the user's query. You can use markdowns to format your response. You should use bullet points to list the information. Make sure the answer is not short and is informative.
@@ -52,7 +52,7 @@ const basicWebSearchResponsePrompt = `
     Place these citations at the end of that particular sentence. You can cite the same sentence multiple times if it is relevant to the user's query like [number1][number2].
     However you do not need to cite it using the same number. You can use different numbers to cite the same sentence multiple times. The number refers to the number of the search result (passed in the context) used to generate that part of the answer.
 
-    Aything inside the following \`context\` HTML block provided below is for your knowledge returned by the search engine and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
+    Anything inside the following \`context\` HTML block provided below is for your knowledge returned by the search engine and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
     talk about the context in your response. 
 
     <context>
diff --git a/src/agents/wolframAlphaSearchAgent.ts b/src/agents/wolframAlphaSearchAgent.ts
index f810a1e..b80fcf3 100644
--- a/src/agents/wolframAlphaSearchAgent.ts
+++ b/src/agents/wolframAlphaSearchAgent.ts
@@ -43,7 +43,7 @@ Rephrased question:
 const basicWolframAlphaSearchResponsePrompt = `
     You are Perplexica, an AI model who is expert at searching the web and answering user's queries. You are set on focus mode 'Wolfram Alpha', this means you will be searching for information on the web using Wolfram Alpha. It is a computational knowledge engine that can answer factual queries and perform computations.
 
-    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containg a brief description of the content of that page).
+    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containing a brief description of the content of that page).
     You must use this context to answer the user's query in the best way possible. Use an unbaised and journalistic tone in your response. Do not repeat the text.
     You must not tell the user to open any link or visit any website to get the answer. You must provide the answer in the response itself. If the user asks for links you can provide them.
     Your responses should be medium to long in length be informative and relevant to the user's query. You can use markdowns to format your response. You should use bullet points to list the information. Make sure the answer is not short and is informative.
@@ -51,7 +51,7 @@ const basicWolframAlphaSearchResponsePrompt = `
     Place these citations at the end of that particular sentence. You can cite the same sentence multiple times if it is relevant to the user's query like [number1][number2].
     However you do not need to cite it using the same number. You can use different numbers to cite the same sentence multiple times. The number refers to the number of the search result (passed in the context) used to generate that part of the answer.
 
-    Aything inside the following \`context\` HTML block provided below is for your knowledge returned by Wolfram Alpha and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
+    Anything inside the following \`context\` HTML block provided below is for your knowledge returned by Wolfram Alpha and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
     talk about the context in your response. 
 
     <context>
diff --git a/src/agents/youtubeSearchAgent.ts b/src/agents/youtubeSearchAgent.ts
index 4e82cc7..334f67e 100644
--- a/src/agents/youtubeSearchAgent.ts
+++ b/src/agents/youtubeSearchAgent.ts
@@ -44,7 +44,7 @@ Rephrased question:
 const basicYoutubeSearchResponsePrompt = `
     You are Perplexica, an AI model who is expert at searching the web and answering user's queries. You are set on focus mode 'Youtube', this means you will be searching for videos on the web using Youtube and providing information based on the video's transcript.
 
-    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containg a brief description of the content of that page).
+    Generate a response that is informative and relevant to the user's query based on provided context (the context consits of search results containing a brief description of the content of that page).
     You must use this context to answer the user's query in the best way possible. Use an unbaised and journalistic tone in your response. Do not repeat the text.
     You must not tell the user to open any link or visit any website to get the answer. You must provide the answer in the response itself. If the user asks for links you can provide them.
     Your responses should be medium to long in length be informative and relevant to the user's query. You can use markdowns to format your response. You should use bullet points to list the information. Make sure the answer is not short and is informative.
@@ -52,7 +52,7 @@ const basicYoutubeSearchResponsePrompt = `
     Place these citations at the end of that particular sentence. You can cite the same sentence multiple times if it is relevant to the user's query like [number1][number2].
     However you do not need to cite it using the same number. You can use different numbers to cite the same sentence multiple times. The number refers to the number of the search result (passed in the context) used to generate that part of the answer.
 
-    Aything inside the following \`context\` HTML block provided below is for your knowledge returned by Youtube and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
+    Anything inside the following \`context\` HTML block provided below is for your knowledge returned by Youtube and is not shared by the user. You have to answer question on the basis of it and cite the relevant information from it but you do not have to 
     talk about the context in your response. 
 
     <context>

From f4b58c71575923919936eb441fff581ae2d5eebe Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 6 Jul 2024 15:13:05 +0530
Subject: [PATCH 28/37] feat(dockerfile): revert base image back to slim

---
 backend.dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend.dockerfile b/backend.dockerfile
index 910aae7..4886573 100644
--- a/backend.dockerfile
+++ b/backend.dockerfile
@@ -1,4 +1,4 @@
-FROM node:buster-slim
+FROM node:slim
 
 ARG SEARXNG_API_URL
 

From 3ffb20b777dd037106bdafab043cc582d2a20480 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Mon, 8 Jul 2024 01:31:11 +0530
Subject: [PATCH 29/37] feat(backend): fix type errors

---
 src/agents/suggestionGeneratorAgent.ts | 2 +-
 src/websocket/connectionManager.ts     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/agents/suggestionGeneratorAgent.ts b/src/agents/suggestionGeneratorAgent.ts
index 0efdfa9..6ba255d 100644
--- a/src/agents/suggestionGeneratorAgent.ts
+++ b/src/agents/suggestionGeneratorAgent.ts
@@ -47,7 +47,7 @@ const generateSuggestions = (
   input: SuggestionGeneratorInput,
   llm: BaseChatModel,
 ) => {
-  (llm as ChatOpenAI).temperature = 0;
+  (llm as unknown as ChatOpenAI).temperature = 0;
   const suggestionGeneratorChain = createSuggestionGeneratorChain(llm);
   return suggestionGeneratorChain.invoke(input);
 };
diff --git a/src/websocket/connectionManager.ts b/src/websocket/connectionManager.ts
index 5cb075b..70e20d9 100644
--- a/src/websocket/connectionManager.ts
+++ b/src/websocket/connectionManager.ts
@@ -45,7 +45,7 @@ export const handleConnection = async (
       chatModelProviders[chatModelProvider][chatModel] &&
       chatModelProvider != 'custom_openai'
     ) {
-      llm = chatModelProviders[chatModelProvider][chatModel] as
+      llm = chatModelProviders[chatModelProvider][chatModel] as unknown as
         | BaseChatModel
         | undefined;
     } else if (chatModelProvider == 'custom_openai') {
@@ -56,7 +56,7 @@ export const handleConnection = async (
         configuration: {
           baseURL: searchParams.get('openAIBaseURL'),
         },
-      });
+      }) as unknown as BaseChatModel;
     }
 
     if (

From 3b4b8a8b0227cee590014f99e10aefd9e1176791 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Mon, 8 Jul 2024 15:24:45 +0530
Subject: [PATCH 30/37] feat(providers): add `custom_openai`

---
 src/lib/providers/index.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib/providers/index.ts b/src/lib/providers/index.ts
index 5807f94..41342b5 100644
--- a/src/lib/providers/index.ts
+++ b/src/lib/providers/index.ts
@@ -22,6 +22,8 @@ export const getAvailableChatModelProviders = async () => {
     models[provider] = await chatModelProviders[provider]();
   }
 
+  models['custom_openai'] = {}
+
   return models;
 };
 

From 8539ce82ad4b289956fc9c57bd1fb505aa26451d Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Mon, 8 Jul 2024 15:39:27 +0530
Subject: [PATCH 31/37] feat(providers): fix loading issues

---
 src/lib/providers/groq.ts         |  2 ++
 src/lib/providers/index.ts        | 22 ++++++++++++++--------
 src/lib/providers/ollama.ts       |  6 +++++-
 src/lib/providers/openai.ts       |  6 +++++-
 src/lib/providers/transformers.ts |  2 +-
 5 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/src/lib/providers/groq.ts b/src/lib/providers/groq.ts
index ecdce4d..35bd125 100644
--- a/src/lib/providers/groq.ts
+++ b/src/lib/providers/groq.ts
@@ -5,6 +5,8 @@ import logger from '../../utils/logger';
 export const loadGroqChatModels = async () => {
   const groqApiKey = getGroqApiKey();
 
+  if (!groqApiKey) return {};
+
   try {
     const chatModels = {
       'LLaMA3 8b': new ChatOpenAI(
diff --git a/src/lib/providers/index.ts b/src/lib/providers/index.ts
index 41342b5..b1d4502 100644
--- a/src/lib/providers/index.ts
+++ b/src/lib/providers/index.ts
@@ -1,7 +1,7 @@
 import { loadGroqChatModels } from './groq';
-import { loadOllamaChatModels } from './ollama';
-import { loadOpenAIChatModels, loadOpenAIEmbeddingsModel } from './openai';
-import { loadTransformersEmbeddingsModel } from './transformers';
+import { loadOllamaChatModels, loadOllamaEmbeddingsModels } from './ollama';
+import { loadOpenAIChatModels, loadOpenAIEmbeddingsModels } from './openai';
+import { loadTransformersEmbeddingsModels } from './transformers';
 
 const chatModelProviders = {
   openai: loadOpenAIChatModels,
@@ -10,16 +10,19 @@ const chatModelProviders = {
 };
 
 const embeddingModelProviders = {
-  openai: loadOpenAIEmbeddingsModel,
-  local: loadTransformersEmbeddingsModel,
-  ollama: loadOllamaChatModels,
+  openai: loadOpenAIEmbeddingsModels,
+  local: loadTransformersEmbeddingsModels,
+  ollama: loadOllamaEmbeddingsModels,
 };
 
 export const getAvailableChatModelProviders = async () => {
   const models = {};
 
   for (const provider in chatModelProviders) {
-    models[provider] = await chatModelProviders[provider]();
+    const providerModels = await chatModelProviders[provider]();
+    if (Object.keys(providerModels).length > 0) {
+      models[provider] = providerModels
+    }
   }
 
   models['custom_openai'] = {}
@@ -31,7 +34,10 @@ export const getAvailableEmbeddingModelProviders = async () => {
   const models = {};
 
   for (const provider in embeddingModelProviders) {
-    models[provider] = await embeddingModelProviders[provider]();
+    const providerModels = await embeddingModelProviders[provider]();
+    if (Object.keys(providerModels).length > 0) {
+      models[provider] = providerModels
+    }
   }
 
   return models;
diff --git a/src/lib/providers/ollama.ts b/src/lib/providers/ollama.ts
index febe5e8..b2901ff 100644
--- a/src/lib/providers/ollama.ts
+++ b/src/lib/providers/ollama.ts
@@ -6,6 +6,8 @@ import { ChatOllama } from '@langchain/community/chat_models/ollama';
 export const loadOllamaChatModels = async () => {
   const ollamaEndpoint = getOllamaApiEndpoint();
 
+  if (!ollamaEndpoint) return {};
+
   try {
     const response = await fetch(`${ollamaEndpoint}/api/tags`, {
       headers: {
@@ -31,9 +33,11 @@ export const loadOllamaChatModels = async () => {
   }
 };
 
-export const loadOpenAIEmbeddingsModel = async () => {
+export const loadOllamaEmbeddingsModels = async () => {
   const ollamaEndpoint = getOllamaApiEndpoint();
 
+  if (!ollamaEndpoint) return {};
+
   try {
     const response = await fetch(`${ollamaEndpoint}/api/tags`, {
       headers: {
diff --git a/src/lib/providers/openai.ts b/src/lib/providers/openai.ts
index 705f1a4..afc7ab8 100644
--- a/src/lib/providers/openai.ts
+++ b/src/lib/providers/openai.ts
@@ -5,6 +5,8 @@ import logger from '../../utils/logger';
 export const loadOpenAIChatModels = async () => {
   const openAIApiKey = getOpenaiApiKey();
 
+  if (!openAIApiKey) return {};
+
   try {
     const chatModels = {
       'GPT-3.5 turbo': new ChatOpenAI({
@@ -36,9 +38,11 @@ export const loadOpenAIChatModels = async () => {
   }
 };
 
-export const loadOpenAIEmbeddingsModel = async () => {
+export const loadOpenAIEmbeddingsModels = async () => {
   const openAIApiKey = getOpenaiApiKey();
 
+  if (!openAIApiKey) return {};
+
   try {
     const embeddingModels = {
       'Text embedding 3 small': new OpenAIEmbeddings({
diff --git a/src/lib/providers/transformers.ts b/src/lib/providers/transformers.ts
index 7ef8596..0ec7052 100644
--- a/src/lib/providers/transformers.ts
+++ b/src/lib/providers/transformers.ts
@@ -1,7 +1,7 @@
 import logger from '../../utils/logger';
 import { HuggingFaceTransformersEmbeddings } from '../huggingfaceTransformer';
 
-export const loadTransformersEmbeddingsModel = async () => {
+export const loadTransformersEmbeddingsModels = async () => {
   try {
     const embeddingModels = {
       'BGE Small': new HuggingFaceTransformersEmbeddings({

From 27e6f5b9e1adeda5aeb5665a366d21d03cb07f30 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Tue, 9 Jul 2024 16:21:45 +0530
Subject: [PATCH 32/37] feat(chat-window): unselect unavailable model

---
 ui/components/ChatWindow.tsx | 49 ++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx
index 675df49..b1a87a2 100644
--- a/ui/components/ChatWindow.tsx
+++ b/ui/components/ChatWindow.tsx
@@ -83,6 +83,55 @@ const useSocket = (
             'embeddingModelProvider',
             embeddingModelProvider,
           );
+        } else {
+          const providers = await fetch(
+            `${process.env.NEXT_PUBLIC_API_URL}/models`,
+            {
+              headers: {
+                'Content-Type': 'application/json',
+              },
+            },
+          ).then(async (res) => await res.json());
+
+          const chatModelProviders = providers.chatModelProviders;
+          const embeddingModelProviders = providers.embeddingModelProviders;
+
+          if (
+            Object.keys(chatModelProviders).length > 0 &&
+            !chatModelProviders[chatModelProvider]
+          ) {
+            chatModelProvider = Object.keys(chatModelProviders)[0];
+            localStorage.setItem('chatModelProvider', chatModelProvider);
+          }
+
+          if (
+            chatModelProvider &&
+            !chatModelProviders[chatModelProvider][chatModel]
+          ) {
+            chatModel = Object.keys(chatModelProviders[chatModelProvider])[0];
+            localStorage.setItem('chatModel', chatModel);
+          }
+
+          if (
+            Object.keys(embeddingModelProviders).length > 0 &&
+            !embeddingModelProviders[embeddingModelProvider]
+          ) {
+            embeddingModelProvider = Object.keys(embeddingModelProviders)[0];
+            localStorage.setItem(
+              'embeddingModelProvider',
+              embeddingModelProvider,
+            );
+          }
+
+          if (
+            embeddingModelProvider &&
+            !embeddingModelProviders[embeddingModelProvider][embeddingModel]
+          ) {
+            embeddingModel = Object.keys(
+              embeddingModelProviders[embeddingModelProvider],
+            )[0];
+            localStorage.setItem('embeddingModel', embeddingModel);
+          }
         }
 
         const wsURL = new URL(url);

From fac41d381242141f39ca3bdfd8f15332286d5ed0 Mon Sep 17 00:00:00 2001
From: sjiampojamarn <18257803+sjiampojamarn@users.noreply.github.com>
Date: Thu, 11 Jul 2024 20:35:54 -0700
Subject: [PATCH 33/37] add gemma2-9b-it

---
 src/lib/providers/groq.ts | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/lib/providers/groq.ts b/src/lib/providers/groq.ts
index 35bd125..ac5dc3b 100644
--- a/src/lib/providers/groq.ts
+++ b/src/lib/providers/groq.ts
@@ -49,6 +49,16 @@ export const loadGroqChatModels = async () => {
           baseURL: 'https://api.groq.com/openai/v1',
         },
       ),
+      'Gemma2 9b': new ChatOpenAI(
+        {
+          openAIApiKey: groqApiKey,
+          modelName: 'gemma2-9b-it',
+          temperature: 0.7,
+        },
+        {
+          baseURL: 'https://api.groq.com/openai/v1',
+        },
+      ),
     };
 
     return chatModels;

From e1732b9bf2b5c3668fa3ccd4d85baf0b156ba973 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sun, 14 Jul 2024 12:37:36 +0530
Subject: [PATCH 34/37] feat(chat-window): fix WS connection errors

---
 ui/components/ChatWindow.tsx | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx
index b1a87a2..f2c89a3 100644
--- a/ui/components/ChatWindow.tsx
+++ b/ui/components/ChatWindow.tsx
@@ -194,8 +194,10 @@ const useSocket = (
     }
 
     return () => {
-      ws?.close();
-      console.log('[DEBUG] closed');
+      if (ws?.readyState === 1) {
+        ws?.close();
+        console.log('[DEBUG] closed');
+      }
     };
   }, [ws, url, setIsWSReady, setError]);
 

From f02393dbe9c18cd5e19df43f3698de30fbb418e1 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Mon, 15 Jul 2024 21:20:16 +0530
Subject: [PATCH 35/37] feat(providers): add anthropic

---
 README.md                        |  3 +-
 package.json                     |  1 +
 sample.config.toml               |  1 +
 src/config.ts                    |  3 ++
 src/lib/providers/anthropic.ts   | 39 ++++++++++++++++++++
 src/lib/providers/index.ts       |  8 +++--
 src/routes/config.ts             |  3 ++
 ui/components/SettingsDialog.tsx | 17 +++++++++
 yarn.lock                        | 62 +++++++++++++++++++++++++++++++-
 9 files changed, 132 insertions(+), 5 deletions(-)
 create mode 100644 src/lib/providers/anthropic.ts

diff --git a/README.md b/README.md
index d1388b0..884fad6 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,8 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker.
 
    - `OPENAI`: Your OpenAI API key. **You only need to fill this if you wish to use OpenAI's models**.
    - `OLLAMA`: Your Ollama API URL. You should enter it as `http://host.docker.internal:PORT_NUMBER`. If you installed Ollama on port 11434, use `http://host.docker.internal:11434`. For other ports, adjust accordingly. **You need to fill this if you wish to use Ollama's models instead of OpenAI's**.
-   - `GROQ`: Your Groq API key. **You only need to fill this if you wish to use Groq's hosted models**
+   - `GROQ`: Your Groq API key. **You only need to fill this if you wish to use Groq's hosted models**.
+   - `ANTHROPIC`: Your Anthropic API key. **You only need to fill this if you wish to use Anthropic models**.
 
      **Note**: You can change these after starting Perplexica from the settings dialog.
 
diff --git a/package.json b/package.json
index 4f2bb32..ccdaa8d 100644
--- a/package.json
+++ b/package.json
@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "@iarna/toml": "^2.2.5",
+    "@langchain/anthropic": "^0.2.3",
     "@langchain/community": "^0.2.16",
     "@langchain/openai": "^0.0.25",
     "@xenova/transformers": "^2.17.1",
diff --git a/sample.config.toml b/sample.config.toml
index 8d35666..f6c6943 100644
--- a/sample.config.toml
+++ b/sample.config.toml
@@ -5,6 +5,7 @@ SIMILARITY_MEASURE = "cosine" # "cosine" or "dot"
 [API_KEYS]
 OPENAI = "" # OpenAI API key - sk-1234567890abcdef1234567890abcdef
 GROQ = "" # Groq API key - gsk_1234567890abcdef1234567890abcdef
+ANTHROPIC = "" # Anthropic API key - sk-ant-1234567890abcdef1234567890abcdef
 
 [API_ENDPOINTS]
 SEARXNG = "http://localhost:32768" # SearxNG API URL
diff --git a/src/config.ts b/src/config.ts
index 7c0c7f1..9ebc182 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -12,6 +12,7 @@ interface Config {
   API_KEYS: {
     OPENAI: string;
     GROQ: string;
+    ANTHROPIC: string;
   };
   API_ENDPOINTS: {
     SEARXNG: string;
@@ -37,6 +38,8 @@ export const getOpenaiApiKey = () => loadConfig().API_KEYS.OPENAI;
 
 export const getGroqApiKey = () => loadConfig().API_KEYS.GROQ;
 
+export const getAnthropicApiKey = () => loadConfig().API_KEYS.ANTHROPIC;
+
 export const getSearxngApiEndpoint = () => loadConfig().API_ENDPOINTS.SEARXNG;
 
 export const getOllamaApiEndpoint = () => loadConfig().API_ENDPOINTS.OLLAMA;
diff --git a/src/lib/providers/anthropic.ts b/src/lib/providers/anthropic.ts
new file mode 100644
index 0000000..58cd164
--- /dev/null
+++ b/src/lib/providers/anthropic.ts
@@ -0,0 +1,39 @@
+import { ChatAnthropic } from '@langchain/anthropic';
+import { getAnthropicApiKey } from '../../config';
+import logger from '../../utils/logger';
+
+export const loadAnthropicChatModels = async () => {
+  const anthropicApiKey = getAnthropicApiKey();
+
+  if (!anthropicApiKey) return {};
+
+  try {
+    const chatModels = {
+      'Claude 3.5 Sonnet': new ChatAnthropic({
+        temperature: 0.7,
+        anthropicApiKey: anthropicApiKey,
+        model: 'claude-3-5-sonnet-20240620',
+      }),
+      'Claude 3 Opus': new ChatAnthropic({
+        temperature: 0.7,
+        anthropicApiKey: anthropicApiKey,
+        model: 'claude-3-opus-20240229',
+      }),
+      'Claude 3 Sonnet': new ChatAnthropic({
+        temperature: 0.7,
+        anthropicApiKey: anthropicApiKey,
+        model: 'claude-3-sonnet-20240229',
+      }),
+      'Claude 3 Haiku': new ChatAnthropic({
+        temperature: 0.7,
+        anthropicApiKey: anthropicApiKey,
+        model: 'claude-3-haiku-20240307',
+      }),
+    };
+
+    return chatModels;
+  } catch (err) {
+    logger.error(`Error loading Anthropic models: ${err}`);
+    return {};
+  }
+};
diff --git a/src/lib/providers/index.ts b/src/lib/providers/index.ts
index b1d4502..d919fd4 100644
--- a/src/lib/providers/index.ts
+++ b/src/lib/providers/index.ts
@@ -1,12 +1,14 @@
 import { loadGroqChatModels } from './groq';
 import { loadOllamaChatModels, loadOllamaEmbeddingsModels } from './ollama';
 import { loadOpenAIChatModels, loadOpenAIEmbeddingsModels } from './openai';
+import { loadAnthropicChatModels } from './anthropic';
 import { loadTransformersEmbeddingsModels } from './transformers';
 
 const chatModelProviders = {
   openai: loadOpenAIChatModels,
   groq: loadGroqChatModels,
   ollama: loadOllamaChatModels,
+  anthropic: loadAnthropicChatModels,
 };
 
 const embeddingModelProviders = {
@@ -21,11 +23,11 @@ export const getAvailableChatModelProviders = async () => {
   for (const provider in chatModelProviders) {
     const providerModels = await chatModelProviders[provider]();
     if (Object.keys(providerModels).length > 0) {
-      models[provider] = providerModels
+      models[provider] = providerModels;
     }
   }
 
-  models['custom_openai'] = {}
+  models['custom_openai'] = {};
 
   return models;
 };
@@ -36,7 +38,7 @@ export const getAvailableEmbeddingModelProviders = async () => {
   for (const provider in embeddingModelProviders) {
     const providerModels = await embeddingModelProviders[provider]();
     if (Object.keys(providerModels).length > 0) {
-      models[provider] = providerModels
+      models[provider] = providerModels;
     }
   }
 
diff --git a/src/routes/config.ts b/src/routes/config.ts
index bf13b63..f255560 100644
--- a/src/routes/config.ts
+++ b/src/routes/config.ts
@@ -6,6 +6,7 @@ import {
 import {
   getGroqApiKey,
   getOllamaApiEndpoint,
+  getAnthropicApiKey,
   getOpenaiApiKey,
   updateConfig,
 } from '../config';
@@ -37,6 +38,7 @@ router.get('/', async (_, res) => {
 
   config['openaiApiKey'] = getOpenaiApiKey();
   config['ollamaApiUrl'] = getOllamaApiEndpoint();
+  config['anthropicApiKey'] = getAnthropicApiKey();
   config['groqApiKey'] = getGroqApiKey();
 
   res.status(200).json(config);
@@ -49,6 +51,7 @@ router.post('/', async (req, res) => {
     API_KEYS: {
       OPENAI: config.openaiApiKey,
       GROQ: config.groqApiKey,
+      ANTHROPIC: config.anthropicApiKey,
     },
     API_ENDPOINTS: {
       OLLAMA: config.ollamaApiUrl,
diff --git a/ui/components/SettingsDialog.tsx b/ui/components/SettingsDialog.tsx
index d6ee18d..788469b 100644
--- a/ui/components/SettingsDialog.tsx
+++ b/ui/components/SettingsDialog.tsx
@@ -56,6 +56,7 @@ interface SettingsType {
   };
   openaiApiKey: string;
   groqApiKey: string;
+  anthropicApiKey: string;
   ollamaApiUrl: string;
 }
 
@@ -439,6 +440,22 @@ const SettingsDialog = ({
                         }
                       />
                     </div>
+                    <div className="flex flex-col space-y-1">
+                      <p className="text-black/70 dark:text-white/70 text-sm">
+                        Anthropic API Key
+                      </p>
+                      <Input
+                        type="text"
+                        placeholder="Anthropic API key"
+                        defaultValue={config.anthropicApiKey}
+                        onChange={(e) =>
+                          setConfig({
+                            ...config,
+                            anthropicApiKey: e.target.value,
+                          })
+                        }
+                      />
+                    </div>
                   </div>
                 )}
                 {isLoading && (
diff --git a/yarn.lock b/yarn.lock
index dceddbd..1a25c1c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,6 +2,20 @@
 # yarn lockfile v1
 
 
+"@anthropic-ai/sdk@^0.22.0":
+  version "0.22.0"
+  resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.22.0.tgz#548e4218d9810fd494e595d4e57cb2d46d301a1a"
+  integrity sha512-dv4BCC6FZJw3w66WNLsHlUFjhu19fS1L/5jMPApwhZLa/Oy1j0A2i3RypmDtHEPp4Wwg3aZkSHksp7VzYWjzmw==
+  dependencies:
+    "@types/node" "^18.11.18"
+    "@types/node-fetch" "^2.6.4"
+    abort-controller "^3.0.0"
+    agentkeepalive "^4.2.1"
+    form-data-encoder "1.7.2"
+    formdata-node "^4.3.2"
+    node-fetch "^2.6.7"
+    web-streams-polyfill "^3.2.1"
+
 "@anthropic-ai/sdk@^0.9.1":
   version "0.9.1"
   resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.9.1.tgz#b2d2b7bf05c90dce502c9a2e869066870f69ba88"
@@ -307,6 +321,17 @@
     "@jridgewell/resolve-uri" "^3.0.3"
     "@jridgewell/sourcemap-codec" "^1.4.10"
 
+"@langchain/anthropic@^0.2.3":
+  version "0.2.3"
+  resolved "https://registry.yarnpkg.com/@langchain/anthropic/-/anthropic-0.2.3.tgz#1505da939f47c90e53dfede0407c497b8177bdf0"
+  integrity sha512-f2fqzLGcvsXXUyZ1vl8cgwkKDGLshOGrPuR9hkhGuBG5m91eq755OqPBxWJuS1TFtNU813cXft3xh0MQbxavwg==
+  dependencies:
+    "@anthropic-ai/sdk" "^0.22.0"
+    "@langchain/core" ">=0.2.9 <0.3.0"
+    fast-xml-parser "^4.3.5"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.4"
+
 "@langchain/community@^0.2.16":
   version "0.2.16"
   resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.2.16.tgz#5888baf7fc7ea272c5f91aaa0e71bc444167262d"
@@ -355,6 +380,24 @@
     zod "^3.22.4"
     zod-to-json-schema "^3.22.3"
 
+"@langchain/core@>=0.2.9 <0.3.0":
+  version "0.2.15"
+  resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.2.15.tgz#1bb99ac4fffe935c7ba37edcaa91abfba3c82219"
+  integrity sha512-L096itIBQ5XNsy5BCCPqIQEk/x4rzI+U4BhYT+fDBYtljESshIi/WzXdmiGfY/6MpVjB76jNuaRgMDmo1m9NeQ==
+  dependencies:
+    ansi-styles "^5.0.0"
+    camelcase "6"
+    decamelize "1.2.0"
+    js-tiktoken "^1.0.12"
+    langsmith "~0.1.30"
+    ml-distance "^4.0.0"
+    mustache "^4.2.0"
+    p-queue "^6.6.2"
+    p-retry "4"
+    uuid "^10.0.0"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
 "@langchain/core@~0.1.44", "@langchain/core@~0.1.45":
   version "0.1.52"
   resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.52.tgz#7619310b83ffa841628efe2e1eda873ca714d068"
@@ -1311,6 +1354,13 @@ fast-fifo@^1.1.0, fast-fifo@^1.2.0:
   resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.3.2.tgz#286e31de96eb96d38a97899815740ba2a4f3640c"
   integrity sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==
 
+fast-xml-parser@^4.3.5:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz#341cc98de71e9ba9e651a67f41f1752d1441a501"
+  integrity sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==
+  dependencies:
+    strnum "^1.0.5"
+
 fecha@^4.2.0:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd"
@@ -2342,6 +2392,11 @@ strip-json-comments@~2.0.1:
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
   integrity sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==
 
+strnum@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db"
+  integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==
+
 supports-color@^5.5.0:
   version "5.5.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f"
@@ -2488,6 +2543,11 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==
 
+uuid@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294"
+  integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==
+
 uuid@^9.0.0:
   version "9.0.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30"
@@ -2592,7 +2652,7 @@ zod-to-json-schema@^3.22.3:
   resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.22.5.tgz#3646e81cfc318dbad2a22519e5ce661615418673"
   integrity sha512-+akaPo6a0zpVCCseDed504KBJUQpEW5QZw7RMneNmKw+fGaML1Z9tUNLnHHAC8x6dzVRO1eB2oEMyZRnuBZg7Q==
 
-zod-to-json-schema@^3.22.5:
+zod-to-json-schema@^3.22.4, zod-to-json-schema@^3.22.5:
   version "3.23.1"
   resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.1.tgz#5225925b8ed5fa20096bd99be076c4b29b53d309"
   integrity sha512-oT9INvydob1XV0v1d2IadrR74rLtDInLvDFfAa1CG0Pmg/vxATk7I2gSelfj271mbzeM4Da0uuDQE/Nkj3DWNw==

From 9195cbcce0a1e5058d95e22e8bc79cdf74cd82bf Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 20 Jul 2024 09:26:46 +0530
Subject: [PATCH 36/37] feat(openai): add GPT-4 Omni mini

---
 src/lib/providers/openai.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/lib/providers/openai.ts b/src/lib/providers/openai.ts
index afc7ab8..8673954 100644
--- a/src/lib/providers/openai.ts
+++ b/src/lib/providers/openai.ts
@@ -29,6 +29,11 @@ export const loadOpenAIChatModels = async () => {
         modelName: 'gpt-4o',
         temperature: 0.7,
       }),
+      'GPT-4 omni mini': new ChatOpenAI({
+        openAIApiKey,
+        modelName: 'gpt-4o-mini',
+        temperature: 0.7,
+      }),
     };
 
     return chatModels;

From c0b3a409dd13f08a67978b7d8f8e7a3d72905a5b Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Sat, 20 Jul 2024 09:27:34 +0530
Subject: [PATCH 37/37] feat(package): bump version

---
 package.json    | 2 +-
 ui/package.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index ccdaa8d..de4cd43 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "perplexica-backend",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "license": "MIT",
   "author": "ItzCrazyKns",
   "scripts": {
diff --git a/ui/package.json b/ui/package.json
index bec350a..6d265c6 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "perplexica-frontend",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "license": "MIT",
   "author": "ItzCrazyKns",
   "scripts": {