From d37a1a80207dd5c6895d135654fd011b172897a3 Mon Sep 17 00:00:00 2001 From: ItzCrazyKns Date: Sat, 20 Apr 2024 11:18:52 +0530 Subject: [PATCH] feat(agents): support local LLMs --- CONTRIBUTING.md | 6 +-- README.md | 13 ++++- sample.config.toml | 5 +- src/agents/academicSearchAgent.ts | 2 +- src/agents/imageSearchAgent.ts | 2 +- src/agents/redditSearchAgent.ts | 2 +- src/agents/webSearchAgent.ts | 2 +- src/agents/wolframAlphaSearchAgent.ts | 2 +- src/agents/youtubeSearchAgent.ts | 2 +- src/config.ts | 14 +++++- src/core/agentPicker.ts | 69 --------------------------- src/lib/providers.ts | 58 ++++++++++++++++++++++ src/{core => lib}/searxng.ts | 0 src/websocket/connectionManager.ts | 36 +++++++++----- ui/components/ChatWindow.tsx | 22 +++++++-- 15 files changed, 135 insertions(+), 100 deletions(-) delete mode 100644 src/core/agentPicker.ts create mode 100644 src/lib/providers.ts rename src/{core => lib}/searxng.ts (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af43ae1..c779f91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,16 +9,14 @@ Perplexica's design consists of two main domains: - **Frontend (`ui` directory)**: This is a Next.js application holding all user interface components. It's a self-contained environment that manages everything the user interacts with. - **Backend (root and `src` directory)**: The backend logic is situated in the `src` folder, but the root directory holds the main `package.json` for backend dependency management. -Both the root directory (for backend configurations outside `src`) and the `ui` folder come with an `.env.example` file. These are templates for environment variables that you need to set up manually for the application to run correctly. - ## Setting Up Your Environment Before diving into coding, setting up your local environment is key. Here's what you need to do: ### Backend -1. In the root directory, locate the `.env.example` file. -2. Rename it to `.env` and fill in the necessary environment variables specific to the backend. +1. In the root directory, locate the `sample.config.toml` file. +2. Rename it to `config.toml` and fill in the necessary configuration fields specific to the backend. 3. Run `npm install` to install dependencies. 4. Use `npm run dev` to start the backend in development mode. diff --git a/README.md b/README.md index 942ba9e..428ee30 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ Perplexica is an open-source AI-powered searching tool or an AI-powered search e ## Features +- **Local LLMs**: You can make use local LLMs such as LLama2 and Mixtral using Ollama. + - **Two Main Modes:** - **Copilot Mode:** (In development) Boosts search by generating different queries to find more relevant internet sources. Like normal search instead of just using the context by SearxNG, it visits the top matches and tries to find relevant sources to the user's query directly from the page. - **Normal Mode:** Processes your query and performs a web search. @@ -58,7 +60,14 @@ There are mainly 2 ways of installing Perplexica - With Docker, Without Docker. 4. Rename the `sample.config.toml` file to `config.toml`. For Docker setups, you need only fill in the following fields: - - `OPENAI`: Your OpenAI API key. + - `CHAT_MODEL`: The name of the LLM to use. Example: `llama2` for Ollama users & `gpt-3.5-turbo` for OpenAI users. + - `CHAT_MODEL_PROVIDER`: The chat model provider, either `openai` or `ollama`. Depending upon which provider you use you would have to fill in the following fields: + + - `OPENAI`: Your OpenAI API key. **You only need to fill this if you wish to use OpenAI's models.** + - `OLLAMA`: Your Ollama API URL. **You need to fill this if you wish to use Ollama's models instead of OpenAI's.** + + **Note**: (In development) You can change these and use different models after running Perplexica as well from the settings page. + - `SIMILARITY_MEASURE`: The similarity measure to use (This is filled by default; you can leave it as is if you are unsure about it.) 5. Ensure you are in the directory containing the `docker-compose.yaml` file and execute: @@ -84,7 +93,7 @@ For setups without Docker: ## Upcoming Features - [ ] Finalizing Copilot Mode -- [ ] Adding support for multiple local LLMs and LLM providers such as Anthropic, Google, etc. +- [X] Adding support for local LLMs - [ ] Adding Discover and History Saving features - [x] Introducing various Focus Modes diff --git a/sample.config.toml b/sample.config.toml index 1082184..9f6d927 100644 --- a/sample.config.toml +++ b/sample.config.toml @@ -1,9 +1,12 @@ [GENERAL] PORT = 3001 # Port to run the server on SIMILARITY_MEASURE = "cosine" # "cosine" or "dot" +CHAT_MODEL_PROVIDER = "openai" # "openai" or "ollama" +CHAT_MODEL = "gpt-3.5-turbo" # Name of the model to use [API_KEYS] OPENAI = "sk-1234567890abcdef1234567890abcdef" # OpenAI API key [API_ENDPOINTS] -SEARXNG = "http://localhost:32768" # SearxNG API ULR \ No newline at end of file +SEARXNG = "http://localhost:32768" # SearxNG API ULR +OLLAMA = "http://localhost:11434" # Ollama API URL \ No newline at end of file diff --git a/src/agents/academicSearchAgent.ts b/src/agents/academicSearchAgent.ts index 466088f..e944946 100644 --- a/src/agents/academicSearchAgent.ts +++ b/src/agents/academicSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/agents/imageSearchAgent.ts b/src/agents/imageSearchAgent.ts index 3adf631..3d8570e 100644 --- a/src/agents/imageSearchAgent.ts +++ b/src/agents/imageSearchAgent.ts @@ -7,7 +7,7 @@ import { PromptTemplate } from '@langchain/core/prompts'; import formatChatHistoryAsString from '../utils/formatHistory'; import { BaseMessage } from '@langchain/core/messages'; import { StringOutputParser } from '@langchain/core/output_parsers'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; const imageSearchChainPrompt = ` diff --git a/src/agents/redditSearchAgent.ts b/src/agents/redditSearchAgent.ts index dca3332..9b460da 100644 --- a/src/agents/redditSearchAgent.ts +++ b/src/agents/redditSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/agents/webSearchAgent.ts b/src/agents/webSearchAgent.ts index 66db2a1..4141d0b 100644 --- a/src/agents/webSearchAgent.ts +++ b/src/agents/webSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/agents/wolframAlphaSearchAgent.ts b/src/agents/wolframAlphaSearchAgent.ts index a68110d..cdcd222 100644 --- a/src/agents/wolframAlphaSearchAgent.ts +++ b/src/agents/wolframAlphaSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/agents/youtubeSearchAgent.ts b/src/agents/youtubeSearchAgent.ts index 4ed8b41..9bb24ed 100644 --- a/src/agents/youtubeSearchAgent.ts +++ b/src/agents/youtubeSearchAgent.ts @@ -11,7 +11,7 @@ import { } from '@langchain/core/runnables'; import { StringOutputParser } from '@langchain/core/output_parsers'; import { Document } from '@langchain/core/documents'; -import { searchSearxng } from '../core/searxng'; +import { searchSearxng } from '../lib/searxng'; import type { StreamEvent } from '@langchain/core/tracers/log_stream'; import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import type { Embeddings } from '@langchain/core/embeddings'; diff --git a/src/config.ts b/src/config.ts index c83522f..055e37f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import toml, { JsonMap } from '@iarna/toml'; +import toml from '@iarna/toml'; const configFileName = 'config.toml'; @@ -8,18 +8,21 @@ interface Config { GENERAL: { PORT: number; SIMILARITY_MEASURE: string; + CHAT_MODEL_PROVIDER: string; + CHAT_MODEL: string; }; API_KEYS: { OPENAI: string; }; API_ENDPOINTS: { SEARXNG: string; + OLLAMA: string; }; } const loadConfig = () => toml.parse( - fs.readFileSync(path.join(process.cwd(), `${configFileName}`), 'utf-8'), + fs.readFileSync(path.join(__dirname, `../${configFileName}`), 'utf-8'), ) as any as Config; export const getPort = () => loadConfig().GENERAL.PORT; @@ -27,6 +30,13 @@ export const getPort = () => loadConfig().GENERAL.PORT; export const getSimilarityMeasure = () => loadConfig().GENERAL.SIMILARITY_MEASURE; +export const getChatModelProvider = () => + loadConfig().GENERAL.CHAT_MODEL_PROVIDER; + +export const getChatModel = () => loadConfig().GENERAL.CHAT_MODEL; + export const getOpenaiApiKey = () => loadConfig().API_KEYS.OPENAI; export const getSearxngApiEndpoint = () => loadConfig().API_ENDPOINTS.SEARXNG; + +export const getOllamaApiEndpoint = () => loadConfig().API_ENDPOINTS.OLLAMA; diff --git a/src/core/agentPicker.ts b/src/core/agentPicker.ts deleted file mode 100644 index ff118da..0000000 --- a/src/core/agentPicker.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { z } from 'zod'; -import { OpenAI } from '@langchain/openai'; -import { RunnableSequence } from '@langchain/core/runnables'; -import { StructuredOutputParser } from 'langchain/output_parsers'; -import { PromptTemplate } from '@langchain/core/prompts'; - -const availableAgents = [ - { - name: 'webSearch', - description: - 'It is expert is searching the web for information and answer user queries', - }, - /* { - name: 'academicSearch', - description: - 'It is expert is searching the academic databases for information and answer user queries. It is particularly good at finding research papers and articles on topics like science, engineering, and technology. Use this instead of wolframAlphaSearch if the user query is not mathematical or scientific in nature', - }, - { - name: 'youtubeSearch', - description: - 'This model is expert at finding videos on youtube based on user queries', - }, - { - name: 'wolframAlphaSearch', - description: - 'This model is expert at finding answers to mathematical and scientific questions based on user queries.', - }, - { - name: 'redditSearch', - description: - 'This model is expert at finding posts and discussions on reddit based on user queries', - }, - { - name: 'writingAssistant', - description: - 'If there is no need for searching, this model is expert at generating text based on user queries', - }, */ -]; - -const parser = StructuredOutputParser.fromZodSchema( - z.object({ - agent: z.string().describe('The name of the selected agent'), - }), -); - -const prompt = ` - You are an AI model who is expert at finding suitable agents for user queries. The available agents are: - ${availableAgents.map((agent) => `- ${agent.name}: ${agent.description}`).join('\n')} - - Your task is to find the most suitable agent for the following query: {query} - - {format_instructions} -`; - -const chain = RunnableSequence.from([ - PromptTemplate.fromTemplate(prompt), - new OpenAI({ temperature: 0 }), - parser, -]); - -const pickSuitableAgent = async (query: string) => { - const res = await chain.invoke({ - query, - format_instructions: parser.getFormatInstructions(), - }); - return res.agent; -}; - -export default pickSuitableAgent; diff --git a/src/lib/providers.ts b/src/lib/providers.ts new file mode 100644 index 0000000..2dfde58 --- /dev/null +++ b/src/lib/providers.ts @@ -0,0 +1,58 @@ +import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; +import { ChatOllama } from '@langchain/community/chat_models/ollama'; +import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama'; +import { getOllamaApiEndpoint, getOpenaiApiKey } from '../config'; + +export const getAvailableProviders = async () => { + const openAIApiKey = getOpenaiApiKey(); + const ollamaEndpoint = getOllamaApiEndpoint(); + + const models = {}; + + if (openAIApiKey) { + models['openai'] = { + 'gpt-3.5-turbo': new ChatOpenAI({ + openAIApiKey, + modelName: 'gpt-3.5-turbo', + temperature: 0.7, + }), + 'gpt-4': new ChatOpenAI({ + openAIApiKey, + modelName: 'gpt-4', + temperature: 0.7, + }), + embeddings: new OpenAIEmbeddings({ + openAIApiKey, + modelName: 'text-embedding-3-large', + }), + }; + } + + if (ollamaEndpoint) { + try { + const response = await fetch(`${ollamaEndpoint}/api/tags`); + + const { models: ollamaModels } = (await response.json()) as any; + + models['ollama'] = ollamaModels.reduce((acc, model) => { + acc[model.model] = new ChatOllama({ + baseUrl: ollamaEndpoint, + model: model.model, + temperature: 0.7, + }); + return acc; + }, {}); + + if (Object.keys(models['ollama']).length > 0) { + models['ollama']['embeddings'] = new OllamaEmbeddings({ + baseUrl: ollamaEndpoint, + model: models['ollama'][Object.keys(models['ollama'])[0]].model, + }); + } + } catch (err) { + console.log(err); + } + } + + return models; +}; diff --git a/src/core/searxng.ts b/src/lib/searxng.ts similarity index 100% rename from src/core/searxng.ts rename to src/lib/searxng.ts diff --git a/src/websocket/connectionManager.ts b/src/websocket/connectionManager.ts index 2dc8d59..5b4e3f5 100644 --- a/src/websocket/connectionManager.ts +++ b/src/websocket/connectionManager.ts @@ -1,18 +1,32 @@ import { WebSocket } from 'ws'; import { handleMessage } from './messageHandler'; -import { ChatOpenAI, OpenAIEmbeddings } from '@langchain/openai'; -import { getOpenaiApiKey } from '../config'; +import { getChatModel, getChatModelProvider } from '../config'; +import { getAvailableProviders } from '../lib/providers'; +import { BaseChatModel } from '@langchain/core/language_models/chat_models'; +import type { Embeddings } from '@langchain/core/embeddings'; -export const handleConnection = (ws: WebSocket) => { - const llm = new ChatOpenAI({ - temperature: 0.7, - openAIApiKey: getOpenaiApiKey(), - }); +export const handleConnection = async (ws: WebSocket) => { + const models = await getAvailableProviders(); + const provider = getChatModelProvider(); + const chatModel = getChatModel(); - const embeddings = new OpenAIEmbeddings({ - openAIApiKey: getOpenaiApiKey(), - modelName: 'text-embedding-3-large', - }); + let llm: BaseChatModel | undefined; + let embeddings: Embeddings | undefined; + + if (models[provider] && models[provider][chatModel]) { + llm = models[provider][chatModel] as BaseChatModel | undefined; + embeddings = models[provider].embeddings as Embeddings | undefined; + } + + if (!llm || !embeddings) { + ws.send( + JSON.stringify({ + type: 'error', + data: 'Invalid LLM or embeddings model selected', + }), + ); + ws.close(); + } ws.on( 'message', diff --git a/ui/components/ChatWindow.tsx b/ui/components/ChatWindow.tsx index 94f4f00..4c138ff 100644 --- a/ui/components/ChatWindow.tsx +++ b/ui/components/ChatWindow.tsx @@ -174,13 +174,25 @@ const ChatWindow = () => { )} ) : ( -
-
+
- ) + ); }; export default ChatWindow;