From 321e60b9931d7e497e0698ba3564b52c3d6fa855 Mon Sep 17 00:00:00 2001
From: ItzCrazyKns <kushagra20103@gmail.com>
Date: Tue, 7 May 2024 12:33:44 +0530
Subject: [PATCH] feat(embedding-providers): load separately, add bert & bge

---
 src/lib/providers.ts | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/lib/providers.ts b/src/lib/providers.ts
index cd0ef78..d2e40f0 100644
--- a/src/lib/providers.ts
+++ b/src/lib/providers.ts
@@ -158,15 +158,26 @@ export const getAvailableEmbeddingModelProviders = async () => {
         return acc;
       }, {});
 
-      models['local'] = {
-        'GTE Small': new HuggingFaceTransformersEmbeddings({
-          modelName: 'Xenova/gte-small',
-        }),
-      };
     } catch (err) {
       logger.error(`Error loading Ollama embeddings: ${err}`);
     }
   }
 
+  try {
+    models['local'] = {
+      'BGE Small': new HuggingFaceTransformersEmbeddings({
+        modelName: 'Xenova/bge-small-en-v1.5',
+      }),
+      'GTE Small': new HuggingFaceTransformersEmbeddings({
+        modelName: 'Xenova/gte-small',
+      }),
+      'Bert Multilingual': new HuggingFaceTransformersEmbeddings({
+        modelName: 'Xenova/bert-base-multilingual-uncased'
+      }),
+    };
+  } catch(err) {
+    logger.error(`Error loading local embeddings: ${err}`); 
+  }
+
   return models;
 };