feat(ollama): add keep_alive param
This commit is contained in:
parent
874505cd0e
commit
c650d1c3d9
|
@ -1,6 +1,7 @@
|
||||||
[GENERAL]
|
[GENERAL]
|
||||||
PORT = 3001 # Port to run the server on
|
PORT = 3001 # Port to run the server on
|
||||||
SIMILARITY_MEASURE = "cosine" # "cosine" or "dot"
|
SIMILARITY_MEASURE = "cosine" # "cosine" or "dot"
|
||||||
|
KEEP_ALIVE = "5m" # How long to keep Ollama models loaded into memory. (Instead of using -1 use "-1m")
|
||||||
|
|
||||||
[API_KEYS]
|
[API_KEYS]
|
||||||
OPENAI = "" # OpenAI API key - sk-1234567890abcdef1234567890abcdef
|
OPENAI = "" # OpenAI API key - sk-1234567890abcdef1234567890abcdef
|
||||||
|
|
|
@ -8,6 +8,7 @@ interface Config {
|
||||||
GENERAL: {
|
GENERAL: {
|
||||||
PORT: number;
|
PORT: number;
|
||||||
SIMILARITY_MEASURE: string;
|
SIMILARITY_MEASURE: string;
|
||||||
|
KEEP_ALIVE: string;
|
||||||
};
|
};
|
||||||
API_KEYS: {
|
API_KEYS: {
|
||||||
OPENAI: string;
|
OPENAI: string;
|
||||||
|
@ -34,6 +35,8 @@ export const getPort = () => loadConfig().GENERAL.PORT;
|
||||||
export const getSimilarityMeasure = () =>
|
export const getSimilarityMeasure = () =>
|
||||||
loadConfig().GENERAL.SIMILARITY_MEASURE;
|
loadConfig().GENERAL.SIMILARITY_MEASURE;
|
||||||
|
|
||||||
|
export const getKeepAlive = () => loadConfig().GENERAL.KEEP_ALIVE;
|
||||||
|
|
||||||
export const getOpenaiApiKey = () => loadConfig().API_KEYS.OPENAI;
|
export const getOpenaiApiKey = () => loadConfig().API_KEYS.OPENAI;
|
||||||
|
|
||||||
export const getGroqApiKey = () => loadConfig().API_KEYS.GROQ;
|
export const getGroqApiKey = () => loadConfig().API_KEYS.GROQ;
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama';
|
import { OllamaEmbeddings } from '@langchain/community/embeddings/ollama';
|
||||||
import { getOllamaApiEndpoint } from '../../config';
|
import { getKeepAlive, getOllamaApiEndpoint } from '../../config';
|
||||||
import logger from '../../utils/logger';
|
import logger from '../../utils/logger';
|
||||||
import { ChatOllama } from '@langchain/community/chat_models/ollama';
|
import { ChatOllama } from '@langchain/community/chat_models/ollama';
|
||||||
|
|
||||||
export const loadOllamaChatModels = async () => {
|
export const loadOllamaChatModels = async () => {
|
||||||
const ollamaEndpoint = getOllamaApiEndpoint();
|
const ollamaEndpoint = getOllamaApiEndpoint();
|
||||||
|
const keepAlive = getKeepAlive();
|
||||||
|
|
||||||
if (!ollamaEndpoint) return {};
|
if (!ollamaEndpoint) return {};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
@ -24,6 +25,7 @@ export const loadOllamaChatModels = async () => {
|
||||||
baseUrl: ollamaEndpoint,
|
baseUrl: ollamaEndpoint,
|
||||||
model: model.model,
|
model: model.model,
|
||||||
temperature: 0.7,
|
temperature: 0.7,
|
||||||
|
keepAlive: keepAlive
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue