feat(video-search): add video search
This commit is contained in:
ItzCrazyKns
parent
bb9a2f538d
commit
6e304e7051
|
|
@ -0,0 +1,90 @@
|
|||
import {
|
||||
RunnableSequence,
|
||||
RunnableMap,
|
||||
RunnableLambda,
|
||||
} from '@langchain/core/runnables';
|
||||
import { PromptTemplate } from '@langchain/core/prompts';
|
||||
import formatChatHistoryAsString from '../utils/formatHistory';
|
||||
import { BaseMessage } from '@langchain/core/messages';
|
||||
import { StringOutputParser } from '@langchain/core/output_parsers';
|
||||
import { searchSearxng } from '../lib/searxng';
|
||||
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
|
||||
|
||||
const VideoSearchChainPrompt = `
|
||||
You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search Youtube for videos.
|
||||
You need to make sure the rephrased question agrees with the conversation and is relevant to the conversation.
|
||||
|
||||
Example:
|
||||
1. Follow up question: How does a car work?
|
||||
Rephrased: How does a car work?
|
||||
|
||||
2. Follow up question: What is the theory of relativity?
|
||||
Rephrased: What is theory of relativity
|
||||
|
||||
3. Follow up question: How does an AC work?
|
||||
Rephrased: How does an AC work
|
||||
|
||||
Conversation:
|
||||
{chat_history}
|
||||
|
||||
Follow up question: {query}
|
||||
Rephrased question:
|
||||
`;
|
||||
|
||||
type VideoSearchChainInput = {
|
||||
chat_history: BaseMessage[];
|
||||
query: string;
|
||||
};
|
||||
|
||||
const strParser = new StringOutputParser();
|
||||
|
||||
const createVideoSearchChain = (llm: BaseChatModel) => {
|
||||
return RunnableSequence.from([
|
||||
RunnableMap.from({
|
||||
chat_history: (input: VideoSearchChainInput) => {
|
||||
return formatChatHistoryAsString(input.chat_history);
|
||||
},
|
||||
query: (input: VideoSearchChainInput) => {
|
||||
return input.query;
|
||||
},
|
||||
}),
|
||||
PromptTemplate.fromTemplate(VideoSearchChainPrompt),
|
||||
llm,
|
||||
strParser,
|
||||
RunnableLambda.from(async (input: string) => {
|
||||
const res = await searchSearxng(input, {
|
||||
engines: ['youtube'],
|
||||
});
|
||||
|
||||
const videos = [];
|
||||
|
||||
res.results.forEach((result) => {
|
||||
if (
|
||||
result.thumbnail &&
|
||||
result.url &&
|
||||
result.title &&
|
||||
result.iframe_src
|
||||
) {
|
||||
videos.push({
|
||||
img_src: result.thumbnail,
|
||||
url: result.url,
|
||||
title: result.title,
|
||||
iframe_src: result.iframe_src,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return videos.slice(0, 10);
|
||||
}),
|
||||
]);
|
||||
};
|
||||
|
||||
const handleVideoSearch = (
|
||||
input: VideoSearchChainInput,
|
||||
llm: BaseChatModel,
|
||||
) => {
|
||||
const VideoSearchChain = createVideoSearchChain(llm);
|
||||
return VideoSearchChain.invoke(input);
|
||||
};
|
||||
|
||||
export default handleVideoSearch;
|
||||
|
|
@ -13,8 +13,10 @@ interface SearxngSearchResult {
|
|||
url: string;
|
||||
img_src?: string;
|
||||
thumbnail_src?: string;
|
||||
thumbnail?: string;
|
||||
content?: string;
|
||||
author?: string;
|
||||
iframe_src?: string;
|
||||
}
|
||||
|
||||
export const searchSearxng = async (
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import express from 'express';
|
||||
import imagesRouter from './images';
|
||||
import videosRouter from './videos';
|
||||
import configRouter from './config';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.use('/images', imagesRouter);
|
||||
router.use('/videos', videosRouter);
|
||||
router.use('/config', configRouter);
|
||||
|
||||
export default router;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
import express from 'express';
|
||||
import { BaseChatModel } from '@langchain/core/language_models/chat_models';
|
||||
import { getAvailableProviders } from '../lib/providers';
|
||||
import { getChatModel, getChatModelProvider } from '../config';
|
||||
import { HumanMessage, AIMessage } from '@langchain/core/messages';
|
||||
import logger from '../utils/logger';
|
||||
import handleVideoSearch from '../agents/videoSearchAgent';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.post('/', async (req, res) => {
|
||||
try {
|
||||
let { query, chat_history } = req.body;
|
||||
|
||||
chat_history = chat_history.map((msg: any) => {
|
||||
if (msg.role === 'user') {
|
||||
return new HumanMessage(msg.content);
|
||||
} else if (msg.role === 'assistant') {
|
||||
return new AIMessage(msg.content);
|
||||
}
|
||||
});
|
||||
|
||||
const models = await getAvailableProviders();
|
||||
const provider = getChatModelProvider();
|
||||
const chatModel = getChatModel();
|
||||
|
||||
let llm: BaseChatModel | undefined;
|
||||
|
||||
if (models[provider] && models[provider][chatModel]) {
|
||||
llm = models[provider][chatModel] as BaseChatModel | undefined;
|
||||
}
|
||||
|
||||
if (!llm) {
|
||||
res.status(500).json({ message: 'Invalid LLM model selected' });
|
||||
return;
|
||||
}
|
||||
|
||||
const videos = await handleVideoSearch({ chat_history, query }, llm);
|
||||
|
||||
res.status(200).json({ videos });
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: 'An error has occurred.' });
|
||||
logger.error(`Error in video search: ${err.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -16,6 +16,7 @@ import Copy from './MessageActions/Copy';
|
|||
import Rewrite from './MessageActions/Rewrite';
|
||||
import MessageSources from './MessageSources';
|
||||
import SearchImages from './SearchImages';
|
||||
import SearchVideos from './SearchVideos';
|
||||
|
||||
const MessageBox = ({
|
||||
message,
|
||||
|
|
@ -120,13 +121,10 @@ const MessageBox = ({
|
|||
query={history[messageIndex - 1].content}
|
||||
chat_history={history.slice(0, messageIndex - 1)}
|
||||
/>
|
||||
<div className="border border-dashed border-[#1C1C1C] px-4 py-2 flex flex-row items-center justify-between rounded-lg text-white text-sm w-full">
|
||||
<div className="flex flex-row items-center space-x-2">
|
||||
<VideoIcon size={17} />
|
||||
<p>Search videos</p>
|
||||
</div>
|
||||
<PlusIcon className="text-[#24A0ED]" size={17} />
|
||||
</div>
|
||||
<SearchVideos
|
||||
chat_history={history.slice(0, messageIndex - 1)}
|
||||
query={history[messageIndex - 1].content}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,190 @@
|
|||
/* eslint-disable @next/next/no-img-element */
|
||||
import { PlayCircle, PlayIcon, PlusIcon, VideoIcon } from 'lucide-react';
|
||||
import { useState } from 'react';
|
||||
import Lightbox, { GenericSlide, VideoSlide } from 'yet-another-react-lightbox';
|
||||
import 'yet-another-react-lightbox/styles.css';
|
||||
import { Message } from './ChatWindow';
|
||||
|
||||
type Video = {
|
||||
url: string;
|
||||
img_src: string;
|
||||
title: string;
|
||||
iframe_src: string;
|
||||
};
|
||||
|
||||
declare module 'yet-another-react-lightbox' {
|
||||
export interface VideoSlide extends GenericSlide {
|
||||
type: 'video-slide';
|
||||
src: string;
|
||||
iframe_src: string;
|
||||
}
|
||||
|
||||
interface SlideTypes {
|
||||
'video-slide': VideoSlide;
|
||||
}
|
||||
}
|
||||
|
||||
const Searchvideos = ({
|
||||
query,
|
||||
chat_history,
|
||||
}: {
|
||||
query: string;
|
||||
chat_history: Message[];
|
||||
}) => {
|
||||
const [videos, setVideos] = useState<Video[] | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [open, setOpen] = useState(false);
|
||||
const [slides, setSlides] = useState<VideoSlide[]>([]);
|
||||
|
||||
return (
|
||||
<>
|
||||
{!loading && videos === null && (
|
||||
<button
|
||||
onClick={async () => {
|
||||
setLoading(true);
|
||||
const res = await fetch(
|
||||
`${process.env.NEXT_PUBLIC_API_URL}/videos`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: query,
|
||||
chat_history: chat_history,
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
const videos = data.videos;
|
||||
setVideos(videos);
|
||||
setSlides(
|
||||
videos.map((video: Video) => {
|
||||
return {
|
||||
type: 'video-slide',
|
||||
iframe_src: video.iframe_src,
|
||||
src: video.img_src,
|
||||
};
|
||||
}),
|
||||
);
|
||||
setLoading(false);
|
||||
}}
|
||||
className="border border-dashed border-[#1C1C1C] hover:bg-[#1c1c1c] active:scale-95 duration-200 transition px-4 py-2 flex flex-row items-center justify-between rounded-lg text-white text-sm w-full"
|
||||
>
|
||||
<div className="flex flex-row items-center space-x-2">
|
||||
<VideoIcon size={17} />
|
||||
<p>Search videos</p>
|
||||
</div>
|
||||
<PlusIcon className="text-[#24A0ED]" size={17} />
|
||||
</button>
|
||||
)}
|
||||
{loading && (
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
{[...Array(4)].map((_, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="bg-[#1C1C1C] h-32 w-full rounded-lg animate-pulse aspect-video object-cover"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{videos !== null && videos.length > 0 && (
|
||||
<>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
{videos.length > 4
|
||||
? videos.slice(0, 3).map((video, i) => (
|
||||
<div
|
||||
onClick={() => {
|
||||
setOpen(true);
|
||||
setSlides([
|
||||
slides[i],
|
||||
...slides.slice(0, i),
|
||||
...slides.slice(i + 1),
|
||||
]);
|
||||
}}
|
||||
className="relative transition duration-200 active:scale-95 hover:scale-[1.02] cursor-pointer"
|
||||
key={i}
|
||||
>
|
||||
<img
|
||||
src={video.img_src}
|
||||
alt={video.title}
|
||||
className="relative h-full w-full aspect-video object-cover rounded-lg"
|
||||
/>
|
||||
<div className="absolute bg-black/70 text-white/70 px-2 py-1 flex flex-row items-center space-x-1 bottom-1 right-1 rounded-md">
|
||||
<PlayCircle size={15} />
|
||||
<p className="text-xs">Video</p>
|
||||
</div>
|
||||
</div>
|
||||
))
|
||||
: videos.map((video, i) => (
|
||||
<div
|
||||
onClick={() => {
|
||||
setOpen(true);
|
||||
setSlides([
|
||||
slides[i],
|
||||
...slides.slice(0, i),
|
||||
...slides.slice(i + 1),
|
||||
]);
|
||||
}}
|
||||
className="relative transition duration-200 active:scale-95 hover:scale-[1.02] cursor-pointer"
|
||||
key={i}
|
||||
>
|
||||
<img
|
||||
src={video.img_src}
|
||||
alt={video.title}
|
||||
className="relative h-full w-full aspect-video object-cover rounded-lg"
|
||||
/>
|
||||
<div className="absolute bg-black/70 text-white/70 px-2 py-1 flex flex-row items-center space-x-1 bottom-1 right-1 rounded-md">
|
||||
<PlayCircle size={15} />
|
||||
<p className="text-xs">Video</p>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
{videos.length > 4 && (
|
||||
<button
|
||||
onClick={() => setOpen(true)}
|
||||
className="bg-[#111111] hover:bg-[#1c1c1c] transition duration-200 active:scale-95 hover:scale-[1.02] h-auto w-full rounded-lg flex flex-col justify-between text-white p-2"
|
||||
>
|
||||
<div className="flex flex-row items-center space-x-1">
|
||||
{videos.slice(3, 6).map((video, i) => (
|
||||
<img
|
||||
key={i}
|
||||
src={video.img_src}
|
||||
alt={video.title}
|
||||
className="h-6 w-12 rounded-md lg:h-3 lg:w-6 lg:rounded-sm aspect-video object-cover"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-white/70 text-xs">
|
||||
View {videos.length - 3} more
|
||||
</p>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<Lightbox
|
||||
open={open}
|
||||
close={() => setOpen(false)}
|
||||
slides={slides}
|
||||
render={{
|
||||
slide: ({ slide }) =>
|
||||
slide.type === 'video-slide' ? (
|
||||
<div className="h-full w-full flex flex-row items-center justify-center">
|
||||
<iframe
|
||||
src={slide.iframe_src}
|
||||
className="aspect-video max-h-[95vh] w-[95vw] rounded-2xl md:w-[80vw]"
|
||||
allowFullScreen
|
||||
allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture"
|
||||
/>
|
||||
</div>
|
||||
) : null,
|
||||
}}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default Searchvideos;
|
||||
Loading…
Reference in New Issue