From 29388c9eb041172592c776449e49bf9a6b75ae9c Mon Sep 17 00:00:00 2001 From: asifrahaman13 Date: Tue, 25 Jun 2024 20:22:58 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20wip:=20add=20the=20discover=20fu?= =?UTF-8?q?nctionality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 5 +- src/app.ts | 15 +++++ src/config.ts | 8 +++ src/database/mongodb/mongodb.ts | 12 ++++ src/database/mongodb/schema/discover.ts | 17 +++++ src/routes/discover.ts | 16 +++++ src/routes/index.ts | 2 + src/services/discover.cron.ts | 42 +++++++++++++ ui/app/discover/page.tsx | 84 ++++++++++++++++++++++++- 9 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 src/database/mongodb/mongodb.ts create mode 100644 src/database/mongodb/schema/discover.ts create mode 100644 src/routes/discover.ts create mode 100644 src/services/discover.cron.ts diff --git a/package.json b/package.json index 612da3c..914d64f 100644 --- a/package.json +++ b/package.json @@ -17,19 +17,22 @@ "nodemon": "^3.1.0", "prettier": "^3.2.5", "ts-node": "^10.9.2", - "typescript": "^5.4.3" + "typescript": "^5.5.3" }, "dependencies": { "@iarna/toml": "^2.2.5", "@langchain/openai": "^0.0.25", + "@types/node": "^20.14.8", "@xenova/transformers": "^2.17.1", "axios": "^1.6.8", "compute-cosine-similarity": "^1.1.0", "compute-dot": "^1.1.0", "cors": "^2.8.5", + "cron": "^3.1.7", "dotenv": "^16.4.5", "express": "^4.19.2", "langchain": "^0.1.30", + "mongoose": "^8.4.3", "winston": "^3.13.0", "ws": "^8.17.1", "zod": "^3.22.4" diff --git a/src/app.ts b/src/app.ts index b8c2371..8cf725e 100644 --- a/src/app.ts +++ b/src/app.ts @@ -5,6 +5,10 @@ import http from 'http'; import routes from './routes'; import { getPort } from './config'; import logger from './utils/logger'; +import job from './services/discover.cron'; +import connectToMongo from './database/mongodb/mongodb'; + +job.start(); const port = getPort(); @@ -15,6 +19,17 @@ const corsOptions = { origin: '*', }; + + +(async () => { + try { + await connectToMongo(); + console.log("Connected to MongoDB"); + } catch (error) { + console.error("Error connecting to MongoDB:", error); + } +})(); + app.use(cors(corsOptions)); app.use(express.json()); diff --git a/src/config.ts b/src/config.ts index 7c0c7f1..7ad14c7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -17,6 +17,10 @@ interface Config { SEARXNG: string; OLLAMA: string; }; + DATABASE:{ + MONGODB_URI: string; + DATABASE_NAME: string; + } } type RecursivePartial = { @@ -41,6 +45,10 @@ export const getSearxngApiEndpoint = () => loadConfig().API_ENDPOINTS.SEARXNG; export const getOllamaApiEndpoint = () => loadConfig().API_ENDPOINTS.OLLAMA; +export const getMongodbUri = () => loadConfig().DATABASE.MONGODB_URI; + +export const getDatabaseName = () => loadConfig().DATABASE.DATABASE_NAME; + export const updateConfig = (config: RecursivePartial) => { const currentConfig = loadConfig(); diff --git a/src/database/mongodb/mongodb.ts b/src/database/mongodb/mongodb.ts new file mode 100644 index 0000000..a038abf --- /dev/null +++ b/src/database/mongodb/mongodb.ts @@ -0,0 +1,12 @@ +import { connect } from 'mongoose'; +import { config } from 'dotenv'; +import { getDatabaseName, getMongodbUri } from '../../config'; +config(); +const connectToMongo = async () => { + const MONGODB_URI = getMongodbUri(); + const DATABASE_NAME = getDatabaseName(); + const connection= MONGODB_URI + "/" + DATABASE_NAME; + await connect(connection, {}); +}; + +export default connectToMongo; diff --git a/src/database/mongodb/schema/discover.ts b/src/database/mongodb/schema/discover.ts new file mode 100644 index 0000000..318f2ec --- /dev/null +++ b/src/database/mongodb/schema/discover.ts @@ -0,0 +1,17 @@ +import { Model, model, Schema } from "mongoose"; + +export interface IDiscover extends Document { + url: string; + title: string; + content: string; +} + +const discoverSchema = new Schema({ +url: { type: String, required: true }, + title: { type: String, required: true }, + content: { type: String, required: true }, +}); + +const DiscoverModel: Model = model('Discover', discoverSchema); +export default DiscoverModel; + diff --git a/src/routes/discover.ts b/src/routes/discover.ts new file mode 100644 index 0000000..ba1b651 --- /dev/null +++ b/src/routes/discover.ts @@ -0,0 +1,16 @@ +import express from 'express'; +import DiscoverModel from '../database/mongodb/schema/discover'; + +const router = express.Router(); + +router.get('/', async (req, res) => { + try { + const data = await DiscoverModel.find().exec(); + return res.json(data); + } catch (error) { + console.error(error); + return res.status(500).json({ message: 'Internal Server Error' }); + } +}); + +export default router; diff --git a/src/routes/index.ts b/src/routes/index.ts index 257e677..41b75fd 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -4,6 +4,7 @@ import videosRouter from './videos'; import configRouter from './config'; import modelsRouter from './models'; import suggestionsRouter from './suggestions'; +import discoverRouter from './discover'; const router = express.Router(); @@ -12,5 +13,6 @@ router.use('/videos', videosRouter); router.use('/config', configRouter); router.use('/models', modelsRouter); router.use('/suggestions', suggestionsRouter); +router.use('/discover', discoverRouter); export default router; diff --git a/src/services/discover.cron.ts b/src/services/discover.cron.ts new file mode 100644 index 0000000..b31304b --- /dev/null +++ b/src/services/discover.cron.ts @@ -0,0 +1,42 @@ +import { CronJob } from 'cron'; +import axios from 'axios'; +import { getSearxngApiEndpoint } from '../config'; +import DiscoverModel from '../database/mongodb/schema/discover'; + +interface SearxngSearchResult { + title: string; + url: string; + img_src?: string; + content?: string; + author?: string; +} + +async function discover() { + try { + const searxngURL = getSearxngApiEndpoint(); + + const url = new URL(`${searxngURL}/search?format=json`); + url.searchParams.append('q', 'Latest cool articles and developments in AI and technology'); + + const res = await axios.get(url.toString()); + + const results: SearxngSearchResult[] = res.data.results.map((result: any) => ({ + url: result.url, + title: result.title, + content: result.content + })); + + const limitedResults = results.slice(0, 10); + await DiscoverModel.deleteMany({}); + await DiscoverModel.create(limitedResults); + } catch (err) { + throw new Error('Failed to discover latest articles'); + } +} + +const job = new CronJob('*/60 * * * * *', () => { + discover(); + console.log('This job runs every 30 seconds.'); +}); + +export default job; diff --git a/ui/app/discover/page.tsx b/ui/app/discover/page.tsx index a443a17..a0278cd 100644 --- a/ui/app/discover/page.tsx +++ b/ui/app/discover/page.tsx @@ -1,5 +1,87 @@ +/* eslint-disable @next/next/no-img-element */ +'use client'; +import { useEffect, useState } from 'react'; +import { Eye, Share2, Clock, Feather } from 'lucide-react'; +import axios from 'axios'; + +interface Discover { + id?: string; + title: string; + content: string; + url: string; +} + const Page = () => { - return
page
; + const [discover, setDiscover] = useState(null); + + useEffect(() => { + const fetchData = async () => { + try { + const response = await axios.get( + `${process.env.NEXT_PUBLIC_API_URL}/discover`, + ); + setDiscover(response.data); + } catch (error) { + console.error('Error fetching data:', error); + } + }; + + fetchData(); + }, []); + + return ( + <> +
+
+
+ +

Discover

+
+
+
+ + {discover && + discover?.map((item, index) => ( + <> +
+
+ +
+
+
{item.title}
+
{item.content}
+
+
+
+ +
+
15744
+
+
+
+ +
+
154
+
+
+
+ +
+
15
+
+
+
+
+
+ + ))} +
+ + ); }; export default Page;