projectmoon
/
perplexica
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(agents): embed docs & query together 

Embed documents and query together to reduce the time taken for retrieving the sources ~1 seconds.
This commit is contained in:
ItzCrazyKns 2024-04-24 10:08:40 +05:30
parent 3b66808e7d
commit 99ae8f6998
No known key found for this signature in database
GPG Key ID: 8162927C7CCE3065
6 changed files with 20 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docker-compose.yaml
View File

@ -9,6 +9,7 @@ services:
- 4000:8080 - 4000:8080
networks: networks:
- perplexica-network - perplexica-network
perplexica-backend: perplexica-backend:
build: build:
context: . context: .

9
src/agents/academicSearchAgent.ts
View File

@ -166,11 +166,10 @@ const createBasicAcademicSearchAnsweringChain = (
(doc) => doc.pageContent && doc.pageContent.length > 0, (doc) => doc.pageContent && doc.pageContent.length > 0,
); );
const docEmbeddings = await embeddings.embedDocuments( const [docEmbeddings, queryEmbedding] = await Promise.all([
docsWithContent.map((doc) => doc.pageContent), embeddings.embedDocuments(docsWithContent.map((doc) => doc.pageContent)),
); embeddings.embedQuery(query),
]);
const queryEmbedding = await embeddings.embedQuery(query);
const similarity = docEmbeddings.map((docEmbedding, i) => { const similarity = docEmbeddings.map((docEmbedding, i) => {
const sim = computeSimilarity(queryEmbedding, docEmbedding); const sim = computeSimilarity(queryEmbedding, docEmbedding);

9
src/agents/redditSearchAgent.ts
View File

@ -161,11 +161,10 @@ const createBasicRedditSearchAnsweringChain = (
(doc) => doc.pageContent && doc.pageContent.length > 0, (doc) => doc.pageContent && doc.pageContent.length > 0,
); );
const docEmbeddings = await embeddings.embedDocuments( const [docEmbeddings, queryEmbedding] = await Promise.all([
docsWithContent.map((doc) => doc.pageContent), embeddings.embedDocuments(docsWithContent.map((doc) => doc.pageContent)),
); embeddings.embedQuery(query),
]);
const queryEmbedding = await embeddings.embedQuery(query);
const similarity = docEmbeddings.map((docEmbedding, i) => { const similarity = docEmbeddings.map((docEmbedding, i) => {
const sim = computeSimilarity(queryEmbedding, docEmbedding); const sim = computeSimilarity(queryEmbedding, docEmbedding);

9
src/agents/webSearchAgent.ts
View File

@ -159,11 +159,10 @@ const createBasicWebSearchAnsweringChain = (
(doc) => doc.pageContent && doc.pageContent.length > 0, (doc) => doc.pageContent && doc.pageContent.length > 0,
); );
const docEmbeddings = await embeddings.embedDocuments( const [docEmbeddings, queryEmbedding] = await Promise.all([
docsWithContent.map((doc) => doc.pageContent), embeddings.embedDocuments(docsWithContent.map((doc) => doc.pageContent)),
); embeddings.embedQuery(query),
]);
const queryEmbedding = await embeddings.embedQuery(query);
const similarity = docEmbeddings.map((docEmbedding, i) => { const similarity = docEmbeddings.map((docEmbedding, i) => {
const sim = computeSimilarity(queryEmbedding, docEmbedding); const sim = computeSimilarity(queryEmbedding, docEmbedding);

9
src/agents/youtubeSearchAgent.ts
View File

@ -161,11 +161,10 @@ const createBasicYoutubeSearchAnsweringChain = (
(doc) => doc.pageContent && doc.pageContent.length > 0, (doc) => doc.pageContent && doc.pageContent.length > 0,
); );
const docEmbeddings = await embeddings.embedDocuments( const [docEmbeddings, queryEmbedding] = await Promise.all([
docsWithContent.map((doc) => doc.pageContent), embeddings.embedDocuments(docsWithContent.map((doc) => doc.pageContent)),
); embeddings.embedQuery(query),
]);
const queryEmbedding = await embeddings.embedQuery(query);
const similarity = docEmbeddings.map((docEmbedding, i) => { const similarity = docEmbeddings.map((docEmbedding, i) => {
const sim = computeSimilarity(queryEmbedding, docEmbedding); const sim = computeSimilarity(queryEmbedding, docEmbedding);