feat(linkDocument): add error handling

src/lib/linkDocument.ts

@@ -3,6 +3,7 @@ import { htmlToText } from 'html-to-text';
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
 import { Document } from '@langchain/core/documents';
 import pdfParse from 'pdf-parse';
+import logger from '../utils/logger';
 
 export const getDocumentsFromLinks = async ({ links }: { links: string[] }) => {
   const splitter = new RecursiveCharacterTextSplitter();
@@ -16,6 +17,7 @@ export const getDocumentsFromLinks = async ({ links }: { links: string[] }) => {
           ? link
           : `https://${link}`;
 
+      try {
         const res = await axios.get(link, {
           responseType: 'arraybuffer',
         });
@@ -76,6 +78,20 @@ export const getDocumentsFromLinks = async ({ links }: { links: string[] }) => {
         });
 
         docs.push(...linkDocs);
+      } catch (err) {
+        logger.error(
+          `Error at generating documents from links: ${err.message}`,
+        );
+        docs.push(
+          new Document({
+            pageContent: `Failed to retrieve content from the link: ${err.message}`,
+            metadata: {
+              title: 'Failed to retrieve content',
+              url: link,
+            },
+          }),
+        );
+      }
     }),
   );