feat(video-search): add video search

src/agents/videoSearchAgent.ts

@@ -0,0 +1,90 @@
+import {
+  RunnableSequence,
+  RunnableMap,
+  RunnableLambda,
+} from '@langchain/core/runnables';
+import { PromptTemplate } from '@langchain/core/prompts';
+import formatChatHistoryAsString from '../utils/formatHistory';
+import { BaseMessage } from '@langchain/core/messages';
+import { StringOutputParser } from '@langchain/core/output_parsers';
+import { searchSearxng } from '../lib/searxng';
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+
+const VideoSearchChainPrompt = `
+  You will be given a conversation below and a follow up question. You need to rephrase the follow-up question so it is a standalone question that can be used by the LLM to search Youtube for videos.
+  You need to make sure the rephrased question agrees with the conversation and is relevant to the conversation.
+  
+  Example:
+  1. Follow up question: How does a car work?
+  Rephrased: How does a car work?
+  
+  2. Follow up question: What is the theory of relativity?
+  Rephrased: What is theory of relativity
+  
+  3. Follow up question: How does an AC work?
+  Rephrased: How does an AC work
+  
+  Conversation:
+  {chat_history}
+  
+  Follow up question: {query}
+  Rephrased question:
+  `;
+
+type VideoSearchChainInput = {
+  chat_history: BaseMessage[];
+  query: string;
+};
+
+const strParser = new StringOutputParser();
+
+const createVideoSearchChain = (llm: BaseChatModel) => {
+  return RunnableSequence.from([
+    RunnableMap.from({
+      chat_history: (input: VideoSearchChainInput) => {
+        return formatChatHistoryAsString(input.chat_history);
+      },
+      query: (input: VideoSearchChainInput) => {
+        return input.query;
+      },
+    }),
+    PromptTemplate.fromTemplate(VideoSearchChainPrompt),
+    llm,
+    strParser,
+    RunnableLambda.from(async (input: string) => {
+      const res = await searchSearxng(input, {
+        engines: ['youtube'],
+      });
+
+      const videos = [];
+
+      res.results.forEach((result) => {
+        if (
+          result.thumbnail &&
+          result.url &&
+          result.title &&
+          result.iframe_src
+        ) {
+          videos.push({
+            img_src: result.thumbnail,
+            url: result.url,
+            title: result.title,
+            iframe_src: result.iframe_src,
+          });
+        }
+      });
+
+      return videos.slice(0, 10);
+    }),
+  ]);
+};
+
+const handleVideoSearch = (
+  input: VideoSearchChainInput,
+  llm: BaseChatModel,
+) => {
+  const VideoSearchChain = createVideoSearchChain(llm);
+  return VideoSearchChain.invoke(input);
+};
+
+export default handleVideoSearch;

src/lib/searxng.ts

@@ -13,8 +13,10 @@ interface SearxngSearchResult {
   url: string;
   img_src?: string;
   thumbnail_src?: string;
+  thumbnail?: string;
   content?: string;
   author?: string;
+  iframe_src?: string;
 }
 
 export const searchSearxng = async (

src/routes/index.ts

@@ -1,10 +1,12 @@
 import express from 'express';
 import imagesRouter from './images';
+import videosRouter from './videos';
 import configRouter from './config';
 
 const router = express.Router();
 
 router.use('/images', imagesRouter);
+router.use('/videos', videosRouter);
 router.use('/config', configRouter);
 
 export default router;

src/routes/videos.ts

@@ -0,0 +1,47 @@
+import express from 'express';
+import { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { getAvailableProviders } from '../lib/providers';
+import { getChatModel, getChatModelProvider } from '../config';
+import { HumanMessage, AIMessage } from '@langchain/core/messages';
+import logger from '../utils/logger';
+import handleVideoSearch from '../agents/videoSearchAgent';
+
+const router = express.Router();
+
+router.post('/', async (req, res) => {
+  try {
+    let { query, chat_history } = req.body;
+
+    chat_history = chat_history.map((msg: any) => {
+      if (msg.role === 'user') {
+        return new HumanMessage(msg.content);
+      } else if (msg.role === 'assistant') {
+        return new AIMessage(msg.content);
+      }
+    });
+
+    const models = await getAvailableProviders();
+    const provider = getChatModelProvider();
+    const chatModel = getChatModel();
+
+    let llm: BaseChatModel | undefined;
+
+    if (models[provider] && models[provider][chatModel]) {
+      llm = models[provider][chatModel] as BaseChatModel | undefined;
+    }
+
+    if (!llm) {
+      res.status(500).json({ message: 'Invalid LLM model selected' });
+      return;
+    }
+
+    const videos = await handleVideoSearch({ chat_history, query }, llm);
+
+    res.status(200).json({ videos });
+  } catch (err) {
+    res.status(500).json({ message: 'An error has occurred.' });
+    logger.error(`Error in video search: ${err.message}`);
+  }
+});
+
+export default router;

ui/components/MessageBox.tsx

@@ -16,6 +16,7 @@ import Copy from './MessageActions/Copy';
 import Rewrite from './MessageActions/Rewrite';
 import MessageSources from './MessageSources';
 import SearchImages from './SearchImages';
+import SearchVideos from './SearchVideos';
 
 const MessageBox = ({
   message,
@@ -120,13 +121,10 @@ const MessageBox = ({
               query={history[messageIndex - 1].content}
               chat_history={history.slice(0, messageIndex - 1)}
             />
-            <div className="border border-dashed border-[#1C1C1C] px-4 py-2 flex flex-row items-center justify-between rounded-lg text-white text-sm w-full">
+            <SearchVideos
-              <div className="flex flex-row items-center space-x-2">
+              chat_history={history.slice(0, messageIndex - 1)}
-                <VideoIcon size={17} />
+              query={history[messageIndex - 1].content}
-                <p>Search videos</p>
+            />
-              </div>
-              <PlusIcon className="text-[#24A0ED]" size={17} />
-            </div>
           </div>
         </div>
       )}

ui/components/SearchVideos.tsx

@@ -0,0 +1,190 @@
+/* eslint-disable @next/next/no-img-element */
+import { PlayCircle, PlayIcon, PlusIcon, VideoIcon } from 'lucide-react';
+import { useState } from 'react';
+import Lightbox, { GenericSlide, VideoSlide } from 'yet-another-react-lightbox';
+import 'yet-another-react-lightbox/styles.css';
+import { Message } from './ChatWindow';
+
+type Video = {
+  url: string;
+  img_src: string;
+  title: string;
+  iframe_src: string;
+};
+
+declare module 'yet-another-react-lightbox' {
+  export interface VideoSlide extends GenericSlide {
+    type: 'video-slide';
+    src: string;
+    iframe_src: string;
+  }
+
+  interface SlideTypes {
+    'video-slide': VideoSlide;
+  }
+}
+
+const Searchvideos = ({
+  query,
+  chat_history,
+}: {
+  query: string;
+  chat_history: Message[];
+}) => {
+  const [videos, setVideos] = useState<Video[] | null>(null);
+  const [loading, setLoading] = useState(false);
+  const [open, setOpen] = useState(false);
+  const [slides, setSlides] = useState<VideoSlide[]>([]);
+
+  return (
+    <>
+      {!loading && videos === null && (
+        <button
+          onClick={async () => {
+            setLoading(true);
+            const res = await fetch(
+              `${process.env.NEXT_PUBLIC_API_URL}/videos`,
+              {
+                method: 'POST',
+                headers: {
+                  'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                  query: query,
+                  chat_history: chat_history,
+                }),
+              },
+            );
+
+            const data = await res.json();
+
+            const videos = data.videos;
+            setVideos(videos);
+            setSlides(
+              videos.map((video: Video) => {
+                return {
+                  type: 'video-slide',
+                  iframe_src: video.iframe_src,
+                  src: video.img_src,
+                };
+              }),
+            );
+            setLoading(false);
+          }}
+          className="border border-dashed border-[#1C1C1C] hover:bg-[#1c1c1c] active:scale-95 duration-200 transition px-4 py-2 flex flex-row items-center justify-between rounded-lg text-white text-sm w-full"
+        >
+          <div className="flex flex-row items-center space-x-2">
+            <VideoIcon size={17} />
+            <p>Search videos</p>
+          </div>
+          <PlusIcon className="text-[#24A0ED]" size={17} />
+        </button>
+      )}
+      {loading && (
+        <div className="grid grid-cols-2 gap-2">
+          {[...Array(4)].map((_, i) => (
+            <div
+              key={i}
+              className="bg-[#1C1C1C] h-32 w-full rounded-lg animate-pulse aspect-video object-cover"
+            />
+          ))}
+        </div>
+      )}
+      {videos !== null && videos.length > 0 && (
+        <>
+          <div className="grid grid-cols-2 gap-2">
+            {videos.length > 4
+              ? videos.slice(0, 3).map((video, i) => (
+                  <div
+                    onClick={() => {
+                      setOpen(true);
+                      setSlides([
+                        slides[i],
+                        ...slides.slice(0, i),
+                        ...slides.slice(i + 1),
+                      ]);
+                    }}
+                    className="relative transition duration-200 active:scale-95 hover:scale-[1.02] cursor-pointer"
+                    key={i}
+                  >
+                    <img
+                      src={video.img_src}
+                      alt={video.title}
+                      className="relative h-full w-full aspect-video object-cover rounded-lg"
+                    />
+                    <div className="absolute bg-black/70 text-white/70 px-2 py-1 flex flex-row items-center space-x-1 bottom-1 right-1 rounded-md">
+                      <PlayCircle size={15} />
+                      <p className="text-xs">Video</p>
+                    </div>
+                  </div>
+                ))
+              : videos.map((video, i) => (
+                  <div
+                    onClick={() => {
+                      setOpen(true);
+                      setSlides([
+                        slides[i],
+                        ...slides.slice(0, i),
+                        ...slides.slice(i + 1),
+                      ]);
+                    }}
+                    className="relative transition duration-200 active:scale-95 hover:scale-[1.02] cursor-pointer"
+                    key={i}
+                  >
+                    <img
+                      src={video.img_src}
+                      alt={video.title}
+                      className="relative h-full w-full aspect-video object-cover rounded-lg"
+                    />
+                    <div className="absolute bg-black/70 text-white/70 px-2 py-1 flex flex-row items-center space-x-1 bottom-1 right-1 rounded-md">
+                      <PlayCircle size={15} />
+                      <p className="text-xs">Video</p>
+                    </div>
+                  </div>
+                ))}
+            {videos.length > 4 && (
+              <button
+                onClick={() => setOpen(true)}
+                className="bg-[#111111] hover:bg-[#1c1c1c] transition duration-200 active:scale-95 hover:scale-[1.02] h-auto w-full rounded-lg flex flex-col justify-between text-white p-2"
+              >
+                <div className="flex flex-row items-center space-x-1">
+                  {videos.slice(3, 6).map((video, i) => (
+                    <img
+                      key={i}
+                      src={video.img_src}
+                      alt={video.title}
+                      className="h-6 w-12 rounded-md lg:h-3 lg:w-6 lg:rounded-sm aspect-video object-cover"
+                    />
+                  ))}
+                </div>
+                <p className="text-white/70 text-xs">
+                  View {videos.length - 3} more
+                </p>
+              </button>
+            )}
+          </div>
+          <Lightbox
+            open={open}
+            close={() => setOpen(false)}
+            slides={slides}
+            render={{
+              slide: ({ slide }) =>
+                slide.type === 'video-slide' ? (
+                  <div className="h-full w-full flex flex-row items-center justify-center">
+                    <iframe
+                      src={slide.iframe_src}
+                      className="aspect-video max-h-[95vh] w-[95vw] rounded-2xl md:w-[80vw]"
+                      allowFullScreen
+                      allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture"
+                    />
+                  </div>
+                ) : null,
+            }}
+          />
+        </>
+      )}
+    </>
+  );
+};
+
+export default Searchvideos;