🎉 wip: add the discover functionality

package.json

@@ -17,19 +17,22 @@
     "nodemon": "^3.1.0",
     "prettier": "^3.2.5",
     "ts-node": "^10.9.2",
-    "typescript": "^5.4.3"
+    "typescript": "^5.5.3"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.5",
     "@langchain/openai": "^0.0.25",
+    "@types/node": "^20.14.8",
     "@xenova/transformers": "^2.17.1",
     "axios": "^1.6.8",
     "compute-cosine-similarity": "^1.1.0",
     "compute-dot": "^1.1.0",
     "cors": "^2.8.5",
+    "cron": "^3.1.7",
     "dotenv": "^16.4.5",
     "express": "^4.19.2",
     "langchain": "^0.1.30",
+    "mongoose": "^8.4.3",
     "winston": "^3.13.0",
     "ws": "^8.17.1",
     "zod": "^3.22.4"

src/app.ts

@@ -5,6 +5,10 @@ import http from 'http';
 import routes from './routes';
 import { getPort } from './config';
 import logger from './utils/logger';
+import job from './services/discover.cron';
+import connectToMongo from './database/mongodb/mongodb';
+
+job.start();
 
 const port = getPort();
 
@@ -15,6 +19,17 @@ const corsOptions = {
   origin: '*',
 };
 
+
+
+(async () => {
+    try {
+        await connectToMongo();
+        console.log("Connected to MongoDB");
+    } catch (error) {
+        console.error("Error connecting to MongoDB:", error);
+    }
+})();
+
 app.use(cors(corsOptions));
 app.use(express.json());
 

src/config.ts

@@ -17,6 +17,10 @@ interface Config {
     SEARXNG: string;
     OLLAMA: string;
   };
+  DATABASE:{
+    MONGODB_URI: string;
+    DATABASE_NAME: string;
+  }
 }
 
 type RecursivePartial<T> = {
@@ -41,6 +45,10 @@ export const getSearxngApiEndpoint = () => loadConfig().API_ENDPOINTS.SEARXNG;
 
 export const getOllamaApiEndpoint = () => loadConfig().API_ENDPOINTS.OLLAMA;
 
+export const getMongodbUri = () => loadConfig().DATABASE.MONGODB_URI;
+
+export const getDatabaseName = () => loadConfig().DATABASE.DATABASE_NAME;
+
 export const updateConfig = (config: RecursivePartial<Config>) => {
   const currentConfig = loadConfig();
 

src/database/mongodb/mongodb.ts

@@ -0,0 +1,12 @@
+import { connect } from 'mongoose';
+import { config } from 'dotenv';
+import { getDatabaseName, getMongodbUri } from '../../config';
+config();
+const connectToMongo = async () => { 
+  const MONGODB_URI = getMongodbUri();
+  const DATABASE_NAME = getDatabaseName();
+  const connection= MONGODB_URI + "/" + DATABASE_NAME;
+  await connect(connection, {});
+};
+
+export default connectToMongo;

src/database/mongodb/schema/discover.ts

@@ -0,0 +1,17 @@
+import { Model, model, Schema } from "mongoose";
+
+export interface IDiscover extends Document {
+  url: string;
+  title: string;
+  content: string;
+}
+
+const discoverSchema = new Schema<IDiscover>({
+url: { type: String, required: true },
+  title: { type: String, required: true },
+  content: { type: String, required: true },
+});
+
+const DiscoverModel: Model<IDiscover> = model('Discover', discoverSchema);
+export default DiscoverModel;
+

src/routes/discover.ts

@@ -0,0 +1,16 @@
+import express from 'express';
+import DiscoverModel from '../database/mongodb/schema/discover';
+
+const router = express.Router();
+
+router.get('/', async (req, res) => {
+  try {
+    const data = await DiscoverModel.find().exec();
+    return res.json(data);
+  } catch (error) {
+    console.error(error);
+    return res.status(500).json({ message: 'Internal Server Error' });
+  }
+});
+
+export default router;

src/routes/index.ts

@@ -4,6 +4,7 @@ import videosRouter from './videos';
 import configRouter from './config';
 import modelsRouter from './models';
 import suggestionsRouter from './suggestions';
+import discoverRouter from './discover';
 
 const router = express.Router();
 
@@ -12,5 +13,6 @@ router.use('/videos', videosRouter);
 router.use('/config', configRouter);
 router.use('/models', modelsRouter);
 router.use('/suggestions', suggestionsRouter);
+router.use('/discover', discoverRouter);
 
 export default router;

src/services/discover.cron.ts

@@ -0,0 +1,42 @@
+import { CronJob } from 'cron';
+import axios from 'axios';
+import { getSearxngApiEndpoint } from '../config';
+import DiscoverModel from '../database/mongodb/schema/discover';
+
+interface SearxngSearchResult {
+  title: string;
+  url: string;
+  img_src?: string;
+  content?: string;
+  author?: string;
+}
+
+async function discover() {
+  try {
+    const searxngURL = getSearxngApiEndpoint();
+
+    const url = new URL(`${searxngURL}/search?format=json`);
+    url.searchParams.append('q', 'Latest cool articles and developments in AI and technology');
+
+    const res = await axios.get(url.toString());
+
+    const results: SearxngSearchResult[] = res.data.results.map((result: any) => ({
+      url: result.url,
+      title: result.title,
+      content: result.content
+    }));
+
+    const limitedResults = results.slice(0, 10);
+    await DiscoverModel.deleteMany({});
+    await DiscoverModel.create(limitedResults);
+  } catch (err) {
+    throw new Error('Failed to discover latest articles');
+  }
+}
+
+const job = new CronJob('*/60 * * * * *', () => {
+  discover();
+  console.log('This job runs every 30 seconds.');
+});
+
+export default job;

ui/app/discover/page.tsx

@@ -1,5 +1,87 @@
+/* eslint-disable @next/next/no-img-element */
+'use client';
+import { useEffect, useState } from 'react';
+import { Eye, Share2, Clock, Feather } from 'lucide-react';
+import axios from 'axios';
+
+interface Discover {
+  id?: string;
+  title: string;
+  content: string;
+  url: string;
+}
+
 const Page = () => {
-  return <div>page</div>;
+  const [discover, setDiscover] = useState<Discover[] | null>(null);
+
+  useEffect(() => {
+    const fetchData = async () => {
+      try {
+        const response = await axios.get(
+          `${process.env.NEXT_PUBLIC_API_URL}/discover`,
+        );
+        setDiscover(response.data);
+      } catch (error) {
+        console.error('Error fetching data:', error);
+      }
+    };
+
+    fetchData();
+  }, []);
+
+  return (
+    <>
+      <div className="container">
+        <div className="row">
+          <div className="flex items-center">
+            <Feather />
+            <h1 className="text-3xl font-medium p-2">Discover</h1>
+          </div>
+          <hr className="border-t-2 border-[#2B2C2C] my-4 w-full" />
+        </div>
+
+        {discover &&
+          discover?.map((item, index) => (
+            <>
+              <div className="p-4 flex gap-4" key={index}>
+                <div>
+                  <img
+                    src={`https://s2.googleusercontent.com/s2/favicons?domain_url=${item.url}`}
+                    alt=""
+                    className="h-10 w-auto"
+                  />
+                </div>
+                <div>
+                  <div className="text-md font-semibold">{item.title}</div>
+                  <div className="text-[#8D9191]">{item.content}</div>
+                  <div className="flex flex-row gap-6 text-[#8D9191]">
+                    <div className="flex gap-1 justify-center items-center">
+                      <div>
+                        <Eye size={16} />
+                      </div>
+                      <div>15744</div>
+                    </div>
+                    <div className="flex gap-1 justify-center items-center">
+                      <div>
+                        <Share2 size={16} />
+                      </div>
+                      <div>154</div>
+                    </div>
+                    <div className="flex gap-1 justify-center items-center">
+                      <div>
+                        <Clock size={16} />
+                      </div>
+                      <div>15</div>
+                    </div>
+                  </div>
+                </div>
+              </div>
+              <hr className="border-t-1 border-[#2B2C2C] my-1 w-full" />
+            </>
+          ))}
+      </div>
+    </>
+  );
 };
 
 export default Page;