feat(video-search): add video search

This commit is contained in:
ItzCrazyKns 2024-04-30 14:31:32 +05:30
parent bb9a2f538d
commit 6e304e7051
No known key found for this signature in database
GPG Key ID: 8162927C7CCE3065
6 changed files with 336 additions and 7 deletions

View File

@ -0,0 +1,90 @@
import {
RunnableSequence,
RunnableMap,
RunnableLambda,
} from '@langchain/core/runnables';
import { PromptTemplate } from '@langchain/core/prompts';
import formatChatHistoryAsString from '../utils/formatHistory';
import { BaseMessage } from '@langchain/core/messages';
import { StringOutputParser } from '@langchain/core/output_parsers';
import { searchSearxng } from '../lib/searxng';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
const VideoSearchChainPrompt = `
You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search Youtube for videos.
You need to make sure the rephrased question agrees with the conversation and is relevant to the conversation.
Example:
1. Follow up question: How does a car work?
Rephrased: How does a car work?
2. Follow up question: What is the theory of relativity?
Rephrased: What is theory of relativity
3. Follow up question: How does an AC work?
Rephrased: How does an AC work
Conversation:
{chat_history}
Follow up question: {query}
Rephrased question:
`;
type VideoSearchChainInput = {
chat_history: BaseMessage[];
query: string;
};
const strParser = new StringOutputParser();
const createVideoSearchChain = (llm: BaseChatModel) => {
return RunnableSequence.from([
RunnableMap.from({
chat_history: (input: VideoSearchChainInput) => {
return formatChatHistoryAsString(input.chat_history);
},
query: (input: VideoSearchChainInput) => {
return input.query;
},
}),
PromptTemplate.fromTemplate(VideoSearchChainPrompt),
llm,
strParser,
RunnableLambda.from(async (input: string) => {
const res = await searchSearxng(input, {
engines: ['youtube'],
});
const videos = [];
res.results.forEach((result) => {
if (
result.thumbnail &&
result.url &&
result.title &&
result.iframe_src
) {
videos.push({
img_src: result.thumbnail,
url: result.url,
title: result.title,
iframe_src: result.iframe_src,
});
}
});
return videos.slice(0, 10);
}),
]);
};
const handleVideoSearch = (
input: VideoSearchChainInput,
llm: BaseChatModel,
) => {
const VideoSearchChain = createVideoSearchChain(llm);
return VideoSearchChain.invoke(input);
};
export default handleVideoSearch;

View File

@ -13,8 +13,10 @@ interface SearxngSearchResult {
url: string; url: string;
img_src?: string; img_src?: string;
thumbnail_src?: string; thumbnail_src?: string;
thumbnail?: string;
content?: string; content?: string;
author?: string; author?: string;
iframe_src?: string;
} }
export const searchSearxng = async ( export const searchSearxng = async (

View File

@ -1,10 +1,12 @@
import express from 'express'; import express from 'express';
import imagesRouter from './images'; import imagesRouter from './images';
import videosRouter from './videos';
import configRouter from './config'; import configRouter from './config';
const router = express.Router(); const router = express.Router();
router.use('/images', imagesRouter); router.use('/images', imagesRouter);
router.use('/videos', videosRouter);
router.use('/config', configRouter); router.use('/config', configRouter);
export default router; export default router;

47
src/routes/videos.ts Normal file
View File

@ -0,0 +1,47 @@
import express from 'express';
import { BaseChatModel } from '@langchain/core/language_models/chat_models';
import { getAvailableProviders } from '../lib/providers';
import { getChatModel, getChatModelProvider } from '../config';
import { HumanMessage, AIMessage } from '@langchain/core/messages';
import logger from '../utils/logger';
import handleVideoSearch from '../agents/videoSearchAgent';
const router = express.Router();
router.post('/', async (req, res) => {
try {
let { query, chat_history } = req.body;
chat_history = chat_history.map((msg: any) => {
if (msg.role === 'user') {
return new HumanMessage(msg.content);
} else if (msg.role === 'assistant') {
return new AIMessage(msg.content);
}
});
const models = await getAvailableProviders();
const provider = getChatModelProvider();
const chatModel = getChatModel();
let llm: BaseChatModel | undefined;
if (models[provider] && models[provider][chatModel]) {
llm = models[provider][chatModel] as BaseChatModel | undefined;
}
if (!llm) {
res.status(500).json({ message: 'Invalid LLM model selected' });
return;
}
const videos = await handleVideoSearch({ chat_history, query }, llm);
res.status(200).json({ videos });
} catch (err) {
res.status(500).json({ message: 'An error has occurred.' });
logger.error(`Error in video search: ${err.message}`);
}
});
export default router;

View File

@ -16,6 +16,7 @@ import Copy from './MessageActions/Copy';
import Rewrite from './MessageActions/Rewrite'; import Rewrite from './MessageActions/Rewrite';
import MessageSources from './MessageSources'; import MessageSources from './MessageSources';
import SearchImages from './SearchImages'; import SearchImages from './SearchImages';
import SearchVideos from './SearchVideos';
const MessageBox = ({ const MessageBox = ({
message, message,
@ -120,13 +121,10 @@ const MessageBox = ({
query={history[messageIndex - 1].content} query={history[messageIndex - 1].content}
chat_history={history.slice(0, messageIndex - 1)} chat_history={history.slice(0, messageIndex - 1)}
/> />
<div className="border border-dashed border-[#1C1C1C] px-4 py-2 flex flex-row items-center justify-between rounded-lg text-white text-sm w-full"> <SearchVideos
<div className="flex flex-row items-center space-x-2"> chat_history={history.slice(0, messageIndex - 1)}
<VideoIcon size={17} /> query={history[messageIndex - 1].content}
<p>Search videos</p> />
</div>
<PlusIcon className="text-[#24A0ED]" size={17} />
</div>
</div> </div>
</div> </div>
)} )}

View File

@ -0,0 +1,190 @@
/* eslint-disable @next/next/no-img-element */
import { PlayCircle, PlayIcon, PlusIcon, VideoIcon } from 'lucide-react';
import { useState } from 'react';
import Lightbox, { GenericSlide, VideoSlide } from 'yet-another-react-lightbox';
import 'yet-another-react-lightbox/styles.css';
import { Message } from './ChatWindow';
type Video = {
url: string;
img_src: string;
title: string;
iframe_src: string;
};
declare module 'yet-another-react-lightbox' {
export interface VideoSlide extends GenericSlide {
type: 'video-slide';
src: string;
iframe_src: string;
}
interface SlideTypes {
'video-slide': VideoSlide;
}
}
const Searchvideos = ({
query,
chat_history,
}: {
query: string;
chat_history: Message[];
}) => {
const [videos, setVideos] = useState<Video[] | null>(null);
const [loading, setLoading] = useState(false);
const [open, setOpen] = useState(false);
const [slides, setSlides] = useState<VideoSlide[]>([]);
return (
<>
{!loading && videos === null && (
<button
onClick={async () => {
setLoading(true);
const res = await fetch(
`${process.env.NEXT_PUBLIC_API_URL}/videos`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
query: query,
chat_history: chat_history,
}),
},
);
const data = await res.json();
const videos = data.videos;
setVideos(videos);
setSlides(
videos.map((video: Video) => {
return {
type: 'video-slide',
iframe_src: video.iframe_src,
src: video.img_src,
};
}),
);
setLoading(false);
}}
className="border border-dashed border-[#1C1C1C] hover:bg-[#1c1c1c] active:scale-95 duration-200 transition px-4 py-2 flex flex-row items-center justify-between rounded-lg text-white text-sm w-full"
>
<div className="flex flex-row items-center space-x-2">
<VideoIcon size={17} />
<p>Search videos</p>
</div>
<PlusIcon className="text-[#24A0ED]" size={17} />
</button>
)}
{loading && (
<div className="grid grid-cols-2 gap-2">
{[...Array(4)].map((_, i) => (
<div
key={i}
className="bg-[#1C1C1C] h-32 w-full rounded-lg animate-pulse aspect-video object-cover"
/>
))}
</div>
)}
{videos !== null && videos.length > 0 && (
<>
<div className="grid grid-cols-2 gap-2">
{videos.length > 4
? videos.slice(0, 3).map((video, i) => (
<div
onClick={() => {
setOpen(true);
setSlides([
slides[i],
...slides.slice(0, i),
...slides.slice(i + 1),
]);
}}
className="relative transition duration-200 active:scale-95 hover:scale-[1.02] cursor-pointer"
key={i}
>
<img
src={video.img_src}
alt={video.title}
className="relative h-full w-full aspect-video object-cover rounded-lg"
/>
<div className="absolute bg-black/70 text-white/70 px-2 py-1 flex flex-row items-center space-x-1 bottom-1 right-1 rounded-md">
<PlayCircle size={15} />
<p className="text-xs">Video</p>
</div>
</div>
))
: videos.map((video, i) => (
<div
onClick={() => {
setOpen(true);
setSlides([
slides[i],
...slides.slice(0, i),
...slides.slice(i + 1),
]);
}}
className="relative transition duration-200 active:scale-95 hover:scale-[1.02] cursor-pointer"
key={i}
>
<img
src={video.img_src}
alt={video.title}
className="relative h-full w-full aspect-video object-cover rounded-lg"
/>
<div className="absolute bg-black/70 text-white/70 px-2 py-1 flex flex-row items-center space-x-1 bottom-1 right-1 rounded-md">
<PlayCircle size={15} />
<p className="text-xs">Video</p>
</div>
</div>
))}
{videos.length > 4 && (
<button
onClick={() => setOpen(true)}
className="bg-[#111111] hover:bg-[#1c1c1c] transition duration-200 active:scale-95 hover:scale-[1.02] h-auto w-full rounded-lg flex flex-col justify-between text-white p-2"
>
<div className="flex flex-row items-center space-x-1">
{videos.slice(3, 6).map((video, i) => (
<img
key={i}
src={video.img_src}
alt={video.title}
className="h-6 w-12 rounded-md lg:h-3 lg:w-6 lg:rounded-sm aspect-video object-cover"
/>
))}
</div>
<p className="text-white/70 text-xs">
View {videos.length - 3} more
</p>
</button>
)}
</div>
<Lightbox
open={open}
close={() => setOpen(false)}
slides={slides}
render={{
slide: ({ slide }) =>
slide.type === 'video-slide' ? (
<div className="h-full w-full flex flex-row items-center justify-center">
<iframe
src={slide.iframe_src}
className="aspect-video max-h-[95vh] w-[95vw] rounded-2xl md:w-[80vw]"
allowFullScreen
allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture"
/>
</div>
) : null,
}}
/>
</>
)}
</>
);
};
export default Searchvideos;