From 4bc14dbdd0d2eefb1b4ded3262f6d11bef005075 Mon Sep 17 00:00:00 2001
From: AkashJana18 <akashjana663@gmail.com>
Date: Sun, 27 Oct 2024 02:46:27 +0530
Subject: [PATCH 01/42] fix: 1219

---
 frontend/src/conversation/Conversation.tsx | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/frontend/src/conversation/Conversation.tsx b/frontend/src/conversation/Conversation.tsx
index a8326da71..238a1a0ed 100644
--- a/frontend/src/conversation/Conversation.tsx
+++ b/frontend/src/conversation/Conversation.tsx
@@ -302,14 +302,17 @@ export default function Conversation() {
         )}
       </div>
 
-      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 pb-1 sm:w-[62%] h-auto">
-        <div className="flex w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black">
+      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 z-3 sm:w-[62%] h-auto">
+        <div
+          className="flex w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black"
+          style={{ height: '60px' }}
+        >
           <textarea
             id="inputbox"
             ref={inputRef}
             tabIndex={1}
             placeholder={t('inputPlaceholder')}
-            className={`inputbox-style h-16 w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-white pt-5 pb-[22px] text-base leading-tight opacity-100 focus:outline-none dark:bg-raisin-black dark:text-bright-gray`}
+            className={`inputbox-style w-full overflow-y-hidden overflow-x-hidden whitespace-pre-wrap rounded-full bg-transparent py-5 text-base leading-tight opacity-100 focus:outline-none dark:bg-transparent dark:text-bright-gray`}
             onInput={handleInput}
             onKeyDown={(e) => {
               if (e.key === 'Enter' && !e.shiftKey) {

From 2d37083719a3803d6326edce813f5432da60bdd3 Mon Sep 17 00:00:00 2001
From: AkashJana18 <akashjana663@gmail.com>
Date: Sat, 2 Nov 2024 11:22:39 +0530
Subject: [PATCH 02/42] minor changes

---
 frontend/src/conversation/Conversation.tsx | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/frontend/src/conversation/Conversation.tsx b/frontend/src/conversation/Conversation.tsx
index 238a1a0ed..5c8a187bc 100644
--- a/frontend/src/conversation/Conversation.tsx
+++ b/frontend/src/conversation/Conversation.tsx
@@ -303,10 +303,7 @@ export default function Conversation() {
       </div>
 
       <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 z-3 sm:w-[62%] h-auto">
-        <div
-          className="flex w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black"
-          style={{ height: '60px' }}
-        >
+        <div className="flex w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black">
           <textarea
             id="inputbox"
             ref={inputRef}

From fe16743d163cedb4cb8d0341d73f441ec4e840d2 Mon Sep 17 00:00:00 2001
From: AkashJana18 <akashjana663@gmail.com>
Date: Wed, 6 Nov 2024 12:39:38 +0530
Subject: [PATCH 03/42] fix/scrollbar in input textarea

---
 frontend/src/conversation/Conversation.tsx | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/frontend/src/conversation/Conversation.tsx b/frontend/src/conversation/Conversation.tsx
index 5c8a187bc..be0fb040e 100644
--- a/frontend/src/conversation/Conversation.tsx
+++ b/frontend/src/conversation/Conversation.tsx
@@ -1,29 +1,30 @@
 import { Fragment, useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
-import newChatIcon from '../assets/openNewChat.svg';
+import { useNavigate } from 'react-router-dom';
+import Hero from '../Hero';
 import ArrowDown from '../assets/arrow-down.svg';
+import newChatIcon from '../assets/openNewChat.svg';
 import Send from '../assets/send.svg';
 import SendDark from '../assets/send_dark.svg';
+import ShareIcon from '../assets/share.svg';
 import SpinnerDark from '../assets/spinner-dark.svg';
 import Spinner from '../assets/spinner.svg';
 import RetryIcon from '../components/RetryIcon';
-import { useNavigate } from 'react-router-dom';
-import Hero from '../Hero';
 import { useDarkTheme, useMediaQuery } from '../hooks';
 import { ShareConversationModal } from '../modals/ShareConversationModal';
-import { setConversation, updateConversationId } from './conversationSlice';
 import { selectConversationId } from '../preferences/preferenceSlice';
 import { AppDispatch } from '../store';
 import ConversationBubble from './ConversationBubble';
 import { handleSendFeedback } from './conversationHandlers';
 import { FEEDBACK, Query } from './conversationModels';
-import ShareIcon from '../assets/share.svg';
 import {
   addQuery,
   fetchAnswer,
   selectQueries,
   selectStatus,
+  setConversation,
+  updateConversationId,
   updateQuery,
 } from './conversationSlice';
 
@@ -309,7 +310,7 @@ export default function Conversation() {
             ref={inputRef}
             tabIndex={1}
             placeholder={t('inputPlaceholder')}
-            className={`inputbox-style w-full overflow-y-hidden overflow-x-hidden whitespace-pre-wrap rounded-full bg-transparent py-5 text-base leading-tight opacity-100 focus:outline-none dark:bg-transparent dark:text-bright-gray`}
+            className={`inputbox-style w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-transparent py-5 text-base leading-tight opacity-100 focus:outline-none dark:bg-transparent dark:text-bright-gray`}
             onInput={handleInput}
             onKeyDown={(e) => {
               if (e.key === 'Enter' && !e.shiftKey) {

From 41cbcbc07f3999d6ced0c6e9f45d7642c3737b15 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:15:35 -0500
Subject: [PATCH 04/42] fixing type on env-sample to .env-template

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index eeecb598d..99baf8111 100644
--- a/README.md
+++ b/README.md
@@ -128,7 +128,7 @@ docker compose -f docker-compose-dev.yaml up -d
 > Make sure you have Python 3.10 or 3.11 installed.
 
 1. Export required environment variables or prepare a `.env` file in the project folder:
-   - Copy [.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) and create `.env`.
+   - Copy [.env-template](https://github.com/arc53/DocsGPT/blob/main/application/.env-template) and create `.env`.
 
 (check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.)
 

From bf0dd6946eb426034ff981a5e486b3ee6c9a501e Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:16:00 -0500
Subject: [PATCH 05/42] add page, row_per_page to api/combine

---
 application/api/user/routes.py | 98 ++++++++++++++++++++++------------
 1 file changed, 65 insertions(+), 33 deletions(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index 8e62683e6..6028dfa2f 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -74,7 +74,8 @@ def post(self):
             )
 
         try:
-            conversations_collection.delete_one({"_id": ObjectId(conversation_id)})
+            conversations_collection.delete_one(
+                {"_id": ObjectId(conversation_id)})
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
         return make_response(jsonify({"success": True}), 200)
@@ -221,7 +222,8 @@ def get(self):
         ids = request.args.get("path")
         if not ids:
             return make_response(
-                jsonify({"success": False, "message": "Missing required fields"}), 400
+                jsonify(
+                    {"success": False, "message": "Missing required fields"}), 400
             )
 
         try:
@@ -244,7 +246,8 @@ def get(self):
         source_id = request.args.get("source_id")
         if not source_id:
             return make_response(
-                jsonify({"success": False, "message": "Missing required fields"}), 400
+                jsonify(
+                    {"success": False, "message": "Missing required fields"}), 400
             )
 
         try:
@@ -255,7 +258,8 @@ def get(self):
                 return make_response(jsonify({"status": "not found"}), 404)
 
             if settings.VECTOR_STORE == "faiss":
-                shutil.rmtree(os.path.join(current_dir, "indexes", str(doc["_id"])))
+                shutil.rmtree(os.path.join(
+                    current_dir, "indexes", str(doc["_id"])))
             else:
                 vectorstore = VectorCreator.create_vectorstore(
                     settings.VECTOR_STORE, source_id=str(doc["_id"])
@@ -305,7 +309,8 @@ def post(self):
         user = secure_filename(request.form["user"])
         job_name = secure_filename(request.form["name"])
         try:
-            save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name)
+            save_dir = os.path.join(
+                current_dir, settings.UPLOAD_FOLDER, user, job_name)
             os.makedirs(save_dir, exist_ok=True)
 
             if len(files) > 1:
@@ -435,6 +440,8 @@ def get(self):
         user = "local"
         sort_field = request.args.get('sort', 'date')  # Default to 'date'
         sort_order = request.args.get('order', "desc")  # Default to 'desc'
+        page_number = request.args.get('page', 1)  # Default to 1
+        rows_per_page = request.args.get('rows', 10)  # Default to 10
         data = [
             {
                 "name": "default",
@@ -447,7 +454,7 @@ def get(self):
         ]
 
         try:
-            for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1):
+            for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order == "asc" else -1):
                 data.append(
                     {
                         "id": str(index["_id"]),
@@ -485,10 +492,14 @@ def get(self):
                         "retriever": "brave_search",
                     }
                 )
+                
+            first_index = (int(page_number) - 1) * int(rows_per_page)
+            last_index = first_index + int(rows_per_page)
+            paginated_docs = data[first_index:last_index]
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
-        return make_response(jsonify(data), 200)
+        return make_response(jsonify(paginated_data), 200)
 
 
 @user_ns.route("/api/docs_check")
@@ -595,7 +606,8 @@ def get(self):
         try:
             if prompt_id == "default":
                 with open(
-                    os.path.join(current_dir, "prompts", "chat_combine_default.txt"),
+                    os.path.join(current_dir, "prompts",
+                                 "chat_combine_default.txt"),
                     "r",
                 ) as f:
                     chat_combine_template = f.read()
@@ -603,7 +615,8 @@ def get(self):
 
             elif prompt_id == "creative":
                 with open(
-                    os.path.join(current_dir, "prompts", "chat_combine_creative.txt"),
+                    os.path.join(current_dir, "prompts",
+                                 "chat_combine_creative.txt"),
                     "r",
                 ) as f:
                     chat_reduce_creative = f.read()
@@ -611,7 +624,8 @@ def get(self):
 
             elif prompt_id == "strict":
                 with open(
-                    os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r"
+                    os.path.join(current_dir, "prompts",
+                                 "chat_combine_strict.txt"), "r"
                 ) as f:
                     chat_reduce_strict = f.read()
                 return make_response(jsonify({"content": chat_reduce_strict}), 200)
@@ -627,7 +641,8 @@ def get(self):
 class DeletePrompt(Resource):
     delete_prompt_model = api.model(
         "DeletePromptModel",
-        {"id": fields.String(required=True, description="Prompt ID to delete")},
+        {"id": fields.String(
+            required=True, description="Prompt ID to delete")},
     )
 
     @api.expect(delete_prompt_model)
@@ -747,7 +762,8 @@ def post(self):
                 "chunks": data["chunks"],
             }
             if "source" in data and ObjectId.is_valid(data["source"]):
-                new_api_key["source"] = DBRef("sources", ObjectId(data["source"]))
+                new_api_key["source"] = DBRef(
+                    "sources", ObjectId(data["source"]))
             if "retriever" in data:
                 new_api_key["retriever"] = data["retriever"]
 
@@ -763,7 +779,8 @@ def post(self):
 class DeleteApiKey(Resource):
     delete_api_key_model = api.model(
         "DeleteApiKeyModel",
-        {"id": fields.String(required=True, description="API Key ID to delete")},
+        {"id": fields.String(
+            required=True, description="API Key ID to delete")},
     )
 
     @api.expect(delete_api_key_model)
@@ -776,7 +793,8 @@ def post(self):
             return missing_fields
 
         try:
-            result = api_key_collection.delete_one({"_id": ObjectId(data["id"])})
+            result = api_key_collection.delete_one(
+                {"_id": ObjectId(data["id"])})
             if result.deleted_count == 0:
                 return {"success": False, "message": "API Key not found"}, 404
         except Exception as err:
@@ -811,7 +829,8 @@ def post(self):
         is_promptable = request.args.get("isPromptable", type=inputs.boolean)
         if is_promptable is None:
             return make_response(
-                jsonify({"success": False, "message": "isPromptable is required"}), 400
+                jsonify(
+                    {"success": False, "message": "isPromptable is required"}), 400
             )
 
         user = data.get("user", "local")
@@ -975,7 +994,8 @@ def post(self):
                 )
                 return make_response(
                     jsonify(
-                        {"success": True, "identifier": str(explicit_binary.as_uuid())}
+                        {"success": True, "identifier": str(
+                            explicit_binary.as_uuid())}
                     ),
                     201,
                 )
@@ -991,7 +1011,8 @@ def get(self, identifier: str):
             query_uuid = Binary.from_uuid(
                 uuid.UUID(identifier), UuidRepresentation.STANDARD
             )
-            shared = shared_conversations_collections.find_one({"uuid": query_uuid})
+            shared = shared_conversations_collections.find_one(
+                {"uuid": query_uuid})
             conversation_queries = []
 
             if (
@@ -1071,7 +1092,8 @@ def post(self):
 
         try:
             api_key = (
-                api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+                api_key_collection.find_one(
+                    {"_id": ObjectId(api_key_id)})["key"]
                 if api_key_id
                 else None
             )
@@ -1119,7 +1141,8 @@ def post(self):
                     jsonify({"success": False, "message": "Invalid option"}), 400
                 )
             start_date = end_date - datetime.timedelta(days=filter_days)
-            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+            start_date = start_date.replace(
+                hour=0, minute=0, second=0, microsecond=0)
             end_date = end_date.replace(
                 hour=23, minute=59, second=59, microsecond=999999
             )
@@ -1162,11 +1185,14 @@ def post(self):
 
             for entry in message_data:
                 if filter_option == "last_hour":
-                    daily_messages[entry["_id"]["minute"]] = entry["total_messages"]
+                    daily_messages[entry["_id"]["minute"]
+                                   ] = entry["total_messages"]
                 elif filter_option == "last_24_hour":
-                    daily_messages[entry["_id"]["hour"]] = entry["total_messages"]
+                    daily_messages[entry["_id"]["hour"]
+                                   ] = entry["total_messages"]
                 else:
-                    daily_messages[entry["_id"]["day"]] = entry["total_messages"]
+                    daily_messages[entry["_id"]["day"]
+                                   ] = entry["total_messages"]
 
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
@@ -1206,7 +1232,8 @@ def post(self):
 
         try:
             api_key = (
-                api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+                api_key_collection.find_one(
+                    {"_id": ObjectId(api_key_id)})["key"]
                 if api_key_id
                 else None
             )
@@ -1264,7 +1291,8 @@ def post(self):
                     jsonify({"success": False, "message": "Invalid option"}), 400
                 )
             start_date = end_date - datetime.timedelta(days=filter_days)
-            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+            start_date = start_date.replace(
+                hour=0, minute=0, second=0, microsecond=0)
             end_date = end_date.replace(
                 hour=23, minute=59, second=59, microsecond=999999
             )
@@ -1313,11 +1341,14 @@ def post(self):
 
             for entry in token_usage_data:
                 if filter_option == "last_hour":
-                    daily_token_usage[entry["_id"]["minute"]] = entry["total_tokens"]
+                    daily_token_usage[entry["_id"]
+                                      ["minute"]] = entry["total_tokens"]
                 elif filter_option == "last_24_hour":
-                    daily_token_usage[entry["_id"]["hour"]] = entry["total_tokens"]
+                    daily_token_usage[entry["_id"]
+                                      ["hour"]] = entry["total_tokens"]
                 else:
-                    daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"]
+                    daily_token_usage[entry["_id"]
+                                      ["day"]] = entry["total_tokens"]
 
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
@@ -1357,7 +1388,8 @@ def post(self):
 
         try:
             api_key = (
-                api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+                api_key_collection.find_one(
+                    {"_id": ObjectId(api_key_id)})["key"]
                 if api_key_id
                 else None
             )
@@ -1459,7 +1491,8 @@ def post(self):
                     jsonify({"success": False, "message": "Invalid option"}), 400
                 )
             start_date = end_date - datetime.timedelta(days=filter_days)
-            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+            start_date = start_date.replace(
+                hour=0, minute=0, second=0, microsecond=0)
             end_date = end_date.replace(
                 hour=23, minute=59, second=59, microsecond=999999
             )
@@ -1575,7 +1608,8 @@ def post(self):
 
         try:
             api_key = (
-                api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
+                api_key_collection.find_one(
+                    {"_id": ObjectId(api_key_id)})["key"]
                 if api_key_id
                 else None
             )
@@ -1686,8 +1720,6 @@ def post(self):
         try:
             tts_instance = GoogleTTS()
             audio_base64, detected_language = tts_instance.text_to_speech(text)
-            return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200)
+            return make_response(jsonify({"success": True, 'audio_base64': audio_base64, 'lang': detected_language}), 200)
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
-
-

From d68e731ffd374f26412fa345c71e2f823d086ee8 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:19:16 -0500
Subject: [PATCH 06/42] revert format code

---
 application/api/user/routes.py | 98 ++++++++++++----------------------
 1 file changed, 33 insertions(+), 65 deletions(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index 6028dfa2f..8e62683e6 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -74,8 +74,7 @@ def post(self):
             )
 
         try:
-            conversations_collection.delete_one(
-                {"_id": ObjectId(conversation_id)})
+            conversations_collection.delete_one({"_id": ObjectId(conversation_id)})
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
         return make_response(jsonify({"success": True}), 200)
@@ -222,8 +221,7 @@ def get(self):
         ids = request.args.get("path")
         if not ids:
             return make_response(
-                jsonify(
-                    {"success": False, "message": "Missing required fields"}), 400
+                jsonify({"success": False, "message": "Missing required fields"}), 400
             )
 
         try:
@@ -246,8 +244,7 @@ def get(self):
         source_id = request.args.get("source_id")
         if not source_id:
             return make_response(
-                jsonify(
-                    {"success": False, "message": "Missing required fields"}), 400
+                jsonify({"success": False, "message": "Missing required fields"}), 400
             )
 
         try:
@@ -258,8 +255,7 @@ def get(self):
                 return make_response(jsonify({"status": "not found"}), 404)
 
             if settings.VECTOR_STORE == "faiss":
-                shutil.rmtree(os.path.join(
-                    current_dir, "indexes", str(doc["_id"])))
+                shutil.rmtree(os.path.join(current_dir, "indexes", str(doc["_id"])))
             else:
                 vectorstore = VectorCreator.create_vectorstore(
                     settings.VECTOR_STORE, source_id=str(doc["_id"])
@@ -309,8 +305,7 @@ def post(self):
         user = secure_filename(request.form["user"])
         job_name = secure_filename(request.form["name"])
         try:
-            save_dir = os.path.join(
-                current_dir, settings.UPLOAD_FOLDER, user, job_name)
+            save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name)
             os.makedirs(save_dir, exist_ok=True)
 
             if len(files) > 1:
@@ -440,8 +435,6 @@ def get(self):
         user = "local"
         sort_field = request.args.get('sort', 'date')  # Default to 'date'
         sort_order = request.args.get('order', "desc")  # Default to 'desc'
-        page_number = request.args.get('page', 1)  # Default to 1
-        rows_per_page = request.args.get('rows', 10)  # Default to 10
         data = [
             {
                 "name": "default",
@@ -454,7 +447,7 @@ def get(self):
         ]
 
         try:
-            for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order == "asc" else -1):
+            for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1):
                 data.append(
                     {
                         "id": str(index["_id"]),
@@ -492,14 +485,10 @@ def get(self):
                         "retriever": "brave_search",
                     }
                 )
-                
-            first_index = (int(page_number) - 1) * int(rows_per_page)
-            last_index = first_index + int(rows_per_page)
-            paginated_docs = data[first_index:last_index]
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
-        return make_response(jsonify(paginated_data), 200)
+        return make_response(jsonify(data), 200)
 
 
 @user_ns.route("/api/docs_check")
@@ -606,8 +595,7 @@ def get(self):
         try:
             if prompt_id == "default":
                 with open(
-                    os.path.join(current_dir, "prompts",
-                                 "chat_combine_default.txt"),
+                    os.path.join(current_dir, "prompts", "chat_combine_default.txt"),
                     "r",
                 ) as f:
                     chat_combine_template = f.read()
@@ -615,8 +603,7 @@ def get(self):
 
             elif prompt_id == "creative":
                 with open(
-                    os.path.join(current_dir, "prompts",
-                                 "chat_combine_creative.txt"),
+                    os.path.join(current_dir, "prompts", "chat_combine_creative.txt"),
                     "r",
                 ) as f:
                     chat_reduce_creative = f.read()
@@ -624,8 +611,7 @@ def get(self):
 
             elif prompt_id == "strict":
                 with open(
-                    os.path.join(current_dir, "prompts",
-                                 "chat_combine_strict.txt"), "r"
+                    os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r"
                 ) as f:
                     chat_reduce_strict = f.read()
                 return make_response(jsonify({"content": chat_reduce_strict}), 200)
@@ -641,8 +627,7 @@ def get(self):
 class DeletePrompt(Resource):
     delete_prompt_model = api.model(
         "DeletePromptModel",
-        {"id": fields.String(
-            required=True, description="Prompt ID to delete")},
+        {"id": fields.String(required=True, description="Prompt ID to delete")},
     )
 
     @api.expect(delete_prompt_model)
@@ -762,8 +747,7 @@ def post(self):
                 "chunks": data["chunks"],
             }
             if "source" in data and ObjectId.is_valid(data["source"]):
-                new_api_key["source"] = DBRef(
-                    "sources", ObjectId(data["source"]))
+                new_api_key["source"] = DBRef("sources", ObjectId(data["source"]))
             if "retriever" in data:
                 new_api_key["retriever"] = data["retriever"]
 
@@ -779,8 +763,7 @@ def post(self):
 class DeleteApiKey(Resource):
     delete_api_key_model = api.model(
         "DeleteApiKeyModel",
-        {"id": fields.String(
-            required=True, description="API Key ID to delete")},
+        {"id": fields.String(required=True, description="API Key ID to delete")},
     )
 
     @api.expect(delete_api_key_model)
@@ -793,8 +776,7 @@ def post(self):
             return missing_fields
 
         try:
-            result = api_key_collection.delete_one(
-                {"_id": ObjectId(data["id"])})
+            result = api_key_collection.delete_one({"_id": ObjectId(data["id"])})
             if result.deleted_count == 0:
                 return {"success": False, "message": "API Key not found"}, 404
         except Exception as err:
@@ -829,8 +811,7 @@ def post(self):
         is_promptable = request.args.get("isPromptable", type=inputs.boolean)
         if is_promptable is None:
             return make_response(
-                jsonify(
-                    {"success": False, "message": "isPromptable is required"}), 400
+                jsonify({"success": False, "message": "isPromptable is required"}), 400
             )
 
         user = data.get("user", "local")
@@ -994,8 +975,7 @@ def post(self):
                 )
                 return make_response(
                     jsonify(
-                        {"success": True, "identifier": str(
-                            explicit_binary.as_uuid())}
+                        {"success": True, "identifier": str(explicit_binary.as_uuid())}
                     ),
                     201,
                 )
@@ -1011,8 +991,7 @@ def get(self, identifier: str):
             query_uuid = Binary.from_uuid(
                 uuid.UUID(identifier), UuidRepresentation.STANDARD
             )
-            shared = shared_conversations_collections.find_one(
-                {"uuid": query_uuid})
+            shared = shared_conversations_collections.find_one({"uuid": query_uuid})
             conversation_queries = []
 
             if (
@@ -1092,8 +1071,7 @@ def post(self):
 
         try:
             api_key = (
-                api_key_collection.find_one(
-                    {"_id": ObjectId(api_key_id)})["key"]
+                api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
                 if api_key_id
                 else None
             )
@@ -1141,8 +1119,7 @@ def post(self):
                     jsonify({"success": False, "message": "Invalid option"}), 400
                 )
             start_date = end_date - datetime.timedelta(days=filter_days)
-            start_date = start_date.replace(
-                hour=0, minute=0, second=0, microsecond=0)
+            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
             end_date = end_date.replace(
                 hour=23, minute=59, second=59, microsecond=999999
             )
@@ -1185,14 +1162,11 @@ def post(self):
 
             for entry in message_data:
                 if filter_option == "last_hour":
-                    daily_messages[entry["_id"]["minute"]
-                                   ] = entry["total_messages"]
+                    daily_messages[entry["_id"]["minute"]] = entry["total_messages"]
                 elif filter_option == "last_24_hour":
-                    daily_messages[entry["_id"]["hour"]
-                                   ] = entry["total_messages"]
+                    daily_messages[entry["_id"]["hour"]] = entry["total_messages"]
                 else:
-                    daily_messages[entry["_id"]["day"]
-                                   ] = entry["total_messages"]
+                    daily_messages[entry["_id"]["day"]] = entry["total_messages"]
 
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
@@ -1232,8 +1206,7 @@ def post(self):
 
         try:
             api_key = (
-                api_key_collection.find_one(
-                    {"_id": ObjectId(api_key_id)})["key"]
+                api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
                 if api_key_id
                 else None
             )
@@ -1291,8 +1264,7 @@ def post(self):
                     jsonify({"success": False, "message": "Invalid option"}), 400
                 )
             start_date = end_date - datetime.timedelta(days=filter_days)
-            start_date = start_date.replace(
-                hour=0, minute=0, second=0, microsecond=0)
+            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
             end_date = end_date.replace(
                 hour=23, minute=59, second=59, microsecond=999999
             )
@@ -1341,14 +1313,11 @@ def post(self):
 
             for entry in token_usage_data:
                 if filter_option == "last_hour":
-                    daily_token_usage[entry["_id"]
-                                      ["minute"]] = entry["total_tokens"]
+                    daily_token_usage[entry["_id"]["minute"]] = entry["total_tokens"]
                 elif filter_option == "last_24_hour":
-                    daily_token_usage[entry["_id"]
-                                      ["hour"]] = entry["total_tokens"]
+                    daily_token_usage[entry["_id"]["hour"]] = entry["total_tokens"]
                 else:
-                    daily_token_usage[entry["_id"]
-                                      ["day"]] = entry["total_tokens"]
+                    daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"]
 
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
@@ -1388,8 +1357,7 @@ def post(self):
 
         try:
             api_key = (
-                api_key_collection.find_one(
-                    {"_id": ObjectId(api_key_id)})["key"]
+                api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
                 if api_key_id
                 else None
             )
@@ -1491,8 +1459,7 @@ def post(self):
                     jsonify({"success": False, "message": "Invalid option"}), 400
                 )
             start_date = end_date - datetime.timedelta(days=filter_days)
-            start_date = start_date.replace(
-                hour=0, minute=0, second=0, microsecond=0)
+            start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
             end_date = end_date.replace(
                 hour=23, minute=59, second=59, microsecond=999999
             )
@@ -1608,8 +1575,7 @@ def post(self):
 
         try:
             api_key = (
-                api_key_collection.find_one(
-                    {"_id": ObjectId(api_key_id)})["key"]
+                api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"]
                 if api_key_id
                 else None
             )
@@ -1720,6 +1686,8 @@ def post(self):
         try:
             tts_instance = GoogleTTS()
             audio_base64, detected_language = tts_instance.text_to_speech(text)
-            return make_response(jsonify({"success": True, 'audio_base64': audio_base64, 'lang': detected_language}), 200)
+            return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200)
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+

From 101935ae46e0040e08e299f7d4c872d165f9a9a9 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:20:25 -0500
Subject: [PATCH 07/42] add pagination with default code format

---
 application/api/user/routes.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index 8e62683e6..7490d0fbe 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -435,6 +435,8 @@ def get(self):
         user = "local"
         sort_field = request.args.get('sort', 'date')  # Default to 'date'
         sort_order = request.args.get('order', "desc")  # Default to 'desc'
+        page = request.args.get('page', 1) # Default to 1
+        rows_per_page = request.args.get('rows', 10) # Default to 10
         data = [
             {
                 "name": "default",
@@ -485,10 +487,15 @@ def get(self):
                         "retriever": "brave_search",
                     }
                 )
+
+            first_index = (int(page_number) - 1) * int(rows_per_page)
+            last_index = first_index + int(rows_per_page)
+            paginated_docs = data[first_index:last_index]
+
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
-        return make_response(jsonify(data), 200)
+        return make_response(jsonify(paginated_docs), 200)
 
 
 @user_ns.route("/api/docs_check")

From a35be6ae5712de134f31ea06271cd6d53242b299 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:31:41 -0500
Subject: [PATCH 08/42] add arrow icons for pagination

---
 frontend/src/assets/double-arrow-left.svg  | 5 +++++
 frontend/src/assets/double-arrow-right.svg | 5 +++++
 frontend/src/assets/single-left-arrow.svg  | 3 +++
 frontend/src/assets/single-right-arrow.svg | 3 +++
 4 files changed, 16 insertions(+)
 create mode 100644 frontend/src/assets/double-arrow-left.svg
 create mode 100644 frontend/src/assets/double-arrow-right.svg
 create mode 100644 frontend/src/assets/single-left-arrow.svg
 create mode 100644 frontend/src/assets/single-right-arrow.svg

diff --git a/frontend/src/assets/double-arrow-left.svg b/frontend/src/assets/double-arrow-left.svg
new file mode 100644
index 000000000..cab9ff900
--- /dev/null
+++ b/frontend/src/assets/double-arrow-left.svg
@@ -0,0 +1,5 @@
+<svg width="16" height="12" viewBox="0 0 16 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M7.41 10.59L2.83 6L7.41 1.41L6 0L0 6L6 12L7.41 10.59Z" fill="black" fill-opacity="0.54" />
+    <path d="M15.41 10.59L10.83 6L15.41 1.41L14 0L8 6L14 12L15.41 10.59Z" fill="black"
+        fill-opacity="0.54" />
+</svg>
\ No newline at end of file
diff --git a/frontend/src/assets/double-arrow-right.svg b/frontend/src/assets/double-arrow-right.svg
new file mode 100644
index 000000000..0d5167c20
--- /dev/null
+++ b/frontend/src/assets/double-arrow-right.svg
@@ -0,0 +1,5 @@
+<svg width="16" height="12" viewBox="0 0 16 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M8.59 10.59L13.17 6L8.59 1.41L10 0L16 6L10 12L8.59 10.59Z" fill="black"
+        fill-opacity="0.54" />
+    <path d="M0.59 10.59L5.17 6L0.59 1.41L2 0L8 6L2 12L0.59 10.59Z" fill="black" fill-opacity="0.54" />
+</svg>
\ No newline at end of file
diff --git a/frontend/src/assets/single-left-arrow.svg b/frontend/src/assets/single-left-arrow.svg
new file mode 100644
index 000000000..f28b25921
--- /dev/null
+++ b/frontend/src/assets/single-left-arrow.svg
@@ -0,0 +1,3 @@
+<svg width="8" height="12" viewBox="0 0 8 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M7.41 10.59L2.83 6L7.41 1.41L6 0L0 6L6 12L7.41 10.59Z" fill="black" fill-opacity="0.54" />
+</svg>
\ No newline at end of file
diff --git a/frontend/src/assets/single-right-arrow.svg b/frontend/src/assets/single-right-arrow.svg
new file mode 100644
index 000000000..85729e570
--- /dev/null
+++ b/frontend/src/assets/single-right-arrow.svg
@@ -0,0 +1,3 @@
+<svg width="8" height="12" viewBox="0 0 8 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M0.59 1.41L5.17 6L0.59 10.59L2 12L8 6L2 0L0.59 1.41Z" fill="black" fill-opacity="0.54" />
+</svg>
\ No newline at end of file

From 5dae074c95e1e9f8742fbf41a6e991fbec2db6af Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:31:56 -0500
Subject: [PATCH 09/42] pagination component

---
 .../src/components/DocumentPagination.tsx     | 115 ++++++++++++++++++
 1 file changed, 115 insertions(+)
 create mode 100644 frontend/src/components/DocumentPagination.tsx

diff --git a/frontend/src/components/DocumentPagination.tsx b/frontend/src/components/DocumentPagination.tsx
new file mode 100644
index 000000000..b5d82456b
--- /dev/null
+++ b/frontend/src/components/DocumentPagination.tsx
@@ -0,0 +1,115 @@
+import React, { useState } from 'react';
+import SingleArrowLeft from '../assets/single-left-arrow.svg';
+import SingleArrowRight from '../assets/single-right-arrow.svg';
+import DoubleArrowLeft from '../assets/double-arrow-left.svg';
+import DoubleArrowRight from '../assets/double-arrow-right.svg';
+
+interface PaginationProps {
+  currentPage: number;
+  totalPages: number;
+  rowsPerPage: number;
+  onPageChange: (page: number) => void;
+  onRowsPerPageChange: (rows: number) => void;
+}
+
+const Pagination: React.FC<PaginationProps> = ({
+  currentPage,
+  totalPages,
+  rowsPerPage,
+  onPageChange,
+  onRowsPerPageChange,
+}) => {
+  const [rowsPerPageOptions] = useState([5, 10, 15, 20]);
+
+  const handlePreviousPage = () => {
+    if (currentPage > 1) {
+      onPageChange(currentPage - 1);
+    }
+  };
+
+  const handleNextPage = () => {
+    if (currentPage < totalPages) {
+      onPageChange(currentPage + 1);
+    }
+  };
+
+  const handleFirstPage = () => {
+    onPageChange(1);
+  };
+
+  const handleLastPage = () => {
+    onPageChange(totalPages);
+  };
+
+  return (
+    <div className="flex items-center text-xs justify-end gap-4 mt-2 p-2 border-gray-200">
+      <div className="flex items-center gap-2 ">
+        <span className="text-gray-900 dark:text-gray-50">Rows per page:</span>
+        <select
+          value={rowsPerPage}
+          onChange={(e) => onRowsPerPageChange(Number(e.target.value))}
+          className="border border-gray-300 rounded px-2 py-1"
+        >
+          {rowsPerPageOptions.map((option) => (
+            <option key={option} value={option}>
+              {option}
+            </option>
+          ))}
+        </select>
+      </div>
+
+      <div className="text-gray-900 dark:text-gray-50">
+        Page {currentPage} of {totalPages}
+      </div>
+
+      <div className="flex items-center gap-2 text-gray-900 dark:text-gray-50">
+        <button
+          onClick={handleFirstPage}
+          disabled={currentPage === 1}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={DoubleArrowLeft}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handlePreviousPage}
+          disabled={currentPage === 1}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={SingleArrowLeft}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handleNextPage}
+          disabled={currentPage === totalPages}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={SingleArrowRight}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+        <button
+          onClick={handleLastPage}
+          disabled={currentPage === totalPages}
+          className="px-2 py-1 border rounded disabled:opacity-50"
+        >
+          <img
+            src={DoubleArrowRight}
+            alt="arrow"
+            className="dark:invert dark:sepia dark:brightness-200"
+          />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+export default Pagination;

From ec3407df7ebd7380136e597a750d792eebf11b13 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:32:37 -0500
Subject: [PATCH 10/42] add 2 fiels for make request to api/combine

---
 frontend/src/api/services/userService.ts  | 11 +++++++++--
 frontend/src/preferences/preferenceApi.ts |  9 ++++++++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts
index 53b38f50c..d55cc5bd1 100644
--- a/frontend/src/api/services/userService.ts
+++ b/frontend/src/api/services/userService.ts
@@ -2,8 +2,15 @@ import apiClient from '../client';
 import endpoints from '../endpoints';
 
 const userService = {
-  getDocs: (sort = 'date', order = 'desc'): Promise<any> =>
-    apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`),
+  getDocs: (
+    sort = 'date',
+    order = 'desc',
+    pageNumber = 1,
+    rowsPerPage = 10,
+  ): Promise<any> =>
+    apiClient.get(
+      `${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`,
+    ),
   checkDocs: (data: any): Promise<any> =>
     apiClient.post(endpoints.USER.DOCS_CHECK, data),
   getAPIKeys: (): Promise<any> => apiClient.get(endpoints.USER.API_KEYS),
diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts
index af1060b83..226af119a 100644
--- a/frontend/src/preferences/preferenceApi.ts
+++ b/frontend/src/preferences/preferenceApi.ts
@@ -6,9 +6,16 @@ import { Doc } from '../models/misc';
 export async function getDocs(
   sort = 'date',
   order = 'desc',
+  pageNumber = 1,
+  rowsPerPage = 5,
 ): Promise<Doc[] | null> {
   try {
-    const response = await userService.getDocs(sort, order);
+    const response = await userService.getDocs(
+      sort,
+      order,
+      pageNumber,
+      rowsPerPage,
+    );
     const data = await response.json();
 
     const docs: Doc[] = [];

From df2f69e85fab05e02b01ce6657d0ed43af756564 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:33:26 -0500
Subject: [PATCH 11/42] adjust table of document with mx auto, add pagination
 to Documents Component

---
 frontend/src/index.css              |  4 +-
 frontend/src/settings/Documents.tsx | 65 ++++++++++++++++++++++++-----
 2 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/frontend/src/index.css b/frontend/src/index.css
index 4319403e7..090a31ca6 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -50,11 +50,11 @@ body.dark {
 
 @layer components {
   .table-default {
-    @apply block w-max table-auto content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
+    @apply block w-max mx-auto table-fixed content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
   }
 
   .table-default th {
-    @apply p-4 w-[244px] font-normal text-gray-400; /* Remove border-r */
+    @apply p-4 w-[250px] font-normal text-gray-400; /* Remove border-r */
   }
 
   .table-default th:last-child {
diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index f94d1a87b..cbb1048c3 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -1,4 +1,4 @@
-import React, { useState } from 'react';
+import React, { useState, useEffect } from 'react';
 import PropTypes from 'prop-types';
 import { useTranslation } from 'react-i18next';
 import { useDispatch } from 'react-redux';
@@ -14,6 +14,7 @@ import { getDocs } from '../preferences/preferenceApi';
 import { setSourceDocs } from '../preferences/preferenceSlice';
 import Input from '../components/Input';
 import Upload from '../upload/Upload'; // Import the Upload component
+import Pagination from '../components/DocumentPagination';
 
 // Utility function to format numbers
 const formatTokens = (tokens: number): string => {
@@ -47,22 +48,55 @@ const Documents: React.FC<DocumentsProps> = ({
   const [loading, setLoading] = useState(false);
   const [sortField, setSortField] = useState<'date' | 'tokens'>('date');
   const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc');
+  // Pagination
+  const [currentPage, setCurrentPage] = useState<number>(1);
+  const [rowsPerPage, setRowsPerPage] = useState<number>(10);
+  const [totalPages, setTotalPages] = useState<number>(0);
+  const [fetchedDocuments, setFetchedDocuments] = useState<Doc[]>([]);
+  // Filter documents based on the search term
+  const filteredDocuments = documents?.filter((document) =>
+    document.name.toLowerCase().includes(searchTerm.toLowerCase()),
+  );
+  // State for documents
+  const startIndex = (currentPage - 1) * rowsPerPage;
+  const endIndex = startIndex + rowsPerPage;
+  const currentDocuments = filteredDocuments
+    ? filteredDocuments.slice(startIndex, endIndex)
+    : [];
+
   const syncOptions = [
     { label: 'Never', value: 'never' },
     { label: 'Daily', value: 'daily' },
     { label: 'Weekly', value: 'weekly' },
     { label: 'Monthly', value: 'monthly' },
   ];
-  const refreshDocs = (field: 'date' | 'tokens') => {
+
+  useEffect(() => {
+    if (fetchedDocuments && rowsPerPage > 0) {
+      setTotalPages(
+        Math.max(1, Math.ceil(fetchedDocuments.length / rowsPerPage)),
+      );
+    } else {
+      setTotalPages(0);
+    }
+  }, [fetchedDocuments, rowsPerPage]);
+
+  const refreshDocs = (
+    field: 'date' | 'tokens',
+    pageNumber = currentPage,
+    rows = rowsPerPage,
+  ) => {
     if (field === sortField) {
       setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc');
     } else {
       setSortOrder('desc');
       setSortField(field);
     }
-    getDocs(sortField, sortOrder)
+    console.log('refreshDocs', field, sortOrder, pageNumber, rows);
+    getDocs(sortField, sortOrder, pageNumber, rows)
       .then((data) => {
         dispatch(setSourceDocs(data));
+        setFetchedDocuments(data ? data : []);
       })
       .catch((error) => console.error(error))
       .finally(() => {
@@ -85,11 +119,6 @@ const Documents: React.FC<DocumentsProps> = ({
       });
   };
 
-  // Filter documents based on the search term
-  const filteredDocuments = documents?.filter((document) =>
-    document.name.toLowerCase().includes(searchTerm.toLowerCase()),
-  );
-
   return (
     <div className="mt-8">
       <div className="flex flex-col relative">
@@ -154,15 +183,15 @@ const Documents: React.FC<DocumentsProps> = ({
                 </tr>
               </thead>
               <tbody>
-                {!filteredDocuments?.length && (
+                {!currentDocuments?.length && (
                   <tr>
                     <td colSpan={5} className="!p-4">
                       {t('settings.documents.noData')}
                     </td>
                   </tr>
                 )}
-                {filteredDocuments &&
-                  filteredDocuments.map((document, index) => (
+                {Array.isArray(currentDocuments) &&
+                  currentDocuments.map((document, index) => (
                     <tr key={index}>
                       <td>{document.name}</td>
                       <td>{document.date}</td>
@@ -221,6 +250,20 @@ const Documents: React.FC<DocumentsProps> = ({
           </div>
         )}
       </div>
+      <Pagination
+        currentPage={currentPage}
+        totalPages={totalPages}
+        rowsPerPage={rowsPerPage}
+        onPageChange={(page) => {
+          setCurrentPage(page);
+          refreshDocs(sortField, page, rowsPerPage);
+        }}
+        onRowsPerPageChange={(rows) => {
+          setRowsPerPage(rows);
+          setCurrentPage(1);
+          refreshDocs(sortField, 1, rows);
+        }}
+      />
     </div>
   );
 };

From 928303f27b07920aa93c2e7cb46e1ef1401f9d2f Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Wed, 6 Nov 2024 22:36:59 -0500
Subject: [PATCH 12/42] fix small bug name of page

---
 application/api/user/routes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index 7490d0fbe..5295c2ee7 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -488,7 +488,7 @@ def get(self):
                     }
                 )
 
-            first_index = (int(page_number) - 1) * int(rows_per_page)
+            first_index = (int(page) - 1) * int(rows_per_page)
             last_index = first_index + int(rows_per_page)
             paginated_docs = data[first_index:last_index]
 

From 2a68cc998962a49a6a675f44bdc27adb098f4f54 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Thu, 7 Nov 2024 23:21:06 -0500
Subject: [PATCH 13/42] calculate totalPage, totalDoc(futreUse)

---
 application/api/user/routes.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index 5295c2ee7..e5ad68aef 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -2,6 +2,7 @@
 import os
 import shutil
 import uuid
+import math
 
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
@@ -435,8 +436,8 @@ def get(self):
         user = "local"
         sort_field = request.args.get('sort', 'date')  # Default to 'date'
         sort_order = request.args.get('order', "desc")  # Default to 'desc'
-        page = request.args.get('page', 1) # Default to 1
-        rows_per_page = request.args.get('rows', 10) # Default to 10
+        page = int(request.args.get('page', 1)) # Default to 1
+        rows_per_page = int(request.args.get('rows', 10)) # Default to 10
         data = [
             {
                 "name": "default",
@@ -487,15 +488,23 @@ def get(self):
                         "retriever": "brave_search",
                     }
                 )
+            total_documents = len(data)
+            total_pages = max(1, math.ceil(total_documents / rows_per_page))
 
-            first_index = (int(page) - 1) * int(rows_per_page)
-            last_index = first_index + int(rows_per_page)
+            first_index = (page - 1) * rows_per_page
+            last_index = first_index + rows_per_page
             paginated_docs = data[first_index:last_index]
 
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
-
-        return make_response(jsonify(paginated_docs), 200)
+            
+        response = {
+            "paginated_docs": paginated_docs,
+            "totalDocuments": total_documents,
+            "totalPages": total_pages,
+            "documents":data
+        }
+        return make_response(jsonify(response), 200)
 
 
 @user_ns.route("/api/docs_check")

From 7ff86a2aeee48149ce3aaa388de99e12aab0a0fa Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Thu, 7 Nov 2024 23:21:26 -0500
Subject: [PATCH 14/42] remove default value since provide on getDocs

---
 frontend/src/api/services/userService.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts
index d55cc5bd1..bebadca50 100644
--- a/frontend/src/api/services/userService.ts
+++ b/frontend/src/api/services/userService.ts
@@ -3,10 +3,10 @@ import endpoints from '../endpoints';
 
 const userService = {
   getDocs: (
-    sort = 'date',
-    order = 'desc',
-    pageNumber = 1,
-    rowsPerPage = 10,
+    sort: string,
+    order: string,
+    pageNumber: number,
+    rowsPerPage: number,
   ): Promise<any> =>
     apiClient.get(
       `${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`,

From 3d03826db517b84a374447861c76ed81e024f1d6 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Thu, 7 Nov 2024 23:22:03 -0500
Subject: [PATCH 15/42] add overload function to gettotalpage, and new type

---
 frontend/src/models/misc.ts               |  6 ++++
 frontend/src/preferences/preferenceApi.ts | 40 ++++++++++++++++++-----
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts
index 9affd0ab4..5478722c5 100644
--- a/frontend/src/models/misc.ts
+++ b/frontend/src/models/misc.ts
@@ -14,6 +14,12 @@ export type Doc = {
   syncFrequency?: string;
 };
 
+export type GetDocsResponse = {
+  docs: Doc[];
+  totalDocuments: number;
+  totalPages: number;
+};
+
 export type PromptProps = {
   prompts: { name: string; id: string; type: string }[];
   selectedPrompt: { name: string; id: string; type: string };
diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts
index 226af119a..e77c9e3b2 100644
--- a/frontend/src/preferences/preferenceApi.ts
+++ b/frontend/src/preferences/preferenceApi.ts
@@ -1,14 +1,23 @@
 import conversationService from '../api/services/conversationService';
 import userService from '../api/services/userService';
-import { Doc } from '../models/misc';
+import { Doc, GetDocsResponse } from '../models/misc';
 
 //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
+export async function getDocs(
+  sort?: string,
+  order?: string,
+  pageNumber?: number,
+  rowsPerPage?: number,
+  withPagination?: true,
+): Promise<GetDocsResponse | null>;
+
 export async function getDocs(
   sort = 'date',
   order = 'desc',
   pageNumber = 1,
   rowsPerPage = 5,
-): Promise<Doc[] | null> {
+  withPagination = false,
+): Promise<Doc[] | GetDocsResponse | null> {
   try {
     const response = await userService.getDocs(
       sort,
@@ -17,14 +26,27 @@ export async function getDocs(
       rowsPerPage,
     );
     const data = await response.json();
+    console.log(data);
+    if (withPagination) {
+      const docs: Doc[] = [];
+      Array.isArray(data.paginated_docs) &&
+        data.paginated_docs.forEach((doc: object) => {
+          docs.push(doc as Doc);
+        });
 
-    const docs: Doc[] = [];
-
-    data.forEach((doc: object) => {
-      docs.push(doc as Doc);
-    });
-
-    return docs;
+      const totalDocuments = data.totalDocuments || 0;
+      const totalPages = data.totalPages || 0;
+      console.log(`totalDocuments: ${totalDocuments}`);
+      console.log(`totalPages: ${totalPages}`);
+      return { docs, totalDocuments, totalPages };
+    } else {
+      const docs: Doc[] = [];
+      Array.isArray(data.documents) &&
+        data.documents.forEach((doc: object) => {
+          docs.push(doc as Doc);
+        });
+      return docs;
+    }
   } catch (error) {
     console.log(error);
     return null;

From a2967afb551956f726621d87afc6069eefb0d43c Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Thu, 7 Nov 2024 23:22:28 -0500
Subject: [PATCH 16/42] adjust Setting/Document to use pagination data from
 backend

---
 frontend/src/settings/Documents.tsx | 60 ++++++++++++++---------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index cbb1048c3..819a4638e 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -51,18 +51,15 @@ const Documents: React.FC<DocumentsProps> = ({
   // Pagination
   const [currentPage, setCurrentPage] = useState<number>(1);
   const [rowsPerPage, setRowsPerPage] = useState<number>(10);
-  const [totalPages, setTotalPages] = useState<number>(0);
+  const [totalPages, setTotalPages] = useState<number>(1);
+  const [totalDocuments, setTotalDocuments] = useState<number>(0);
   const [fetchedDocuments, setFetchedDocuments] = useState<Doc[]>([]);
   // Filter documents based on the search term
-  const filteredDocuments = documents?.filter((document) =>
+  const filteredDocuments = fetchedDocuments?.filter((document) =>
     document.name.toLowerCase().includes(searchTerm.toLowerCase()),
   );
   // State for documents
-  const startIndex = (currentPage - 1) * rowsPerPage;
-  const endIndex = startIndex + rowsPerPage;
-  const currentDocuments = filteredDocuments
-    ? filteredDocuments.slice(startIndex, endIndex)
-    : [];
+  const currentDocuments = filteredDocuments ?? [];
 
   const syncOptions = [
     { label: 'Never', value: 'never' },
@@ -71,32 +68,31 @@ const Documents: React.FC<DocumentsProps> = ({
     { label: 'Monthly', value: 'monthly' },
   ];
 
-  useEffect(() => {
-    if (fetchedDocuments && rowsPerPage > 0) {
-      setTotalPages(
-        Math.max(1, Math.ceil(fetchedDocuments.length / rowsPerPage)),
-      );
-    } else {
-      setTotalPages(0);
-    }
-  }, [fetchedDocuments, rowsPerPage]);
-
   const refreshDocs = (
     field: 'date' | 'tokens',
-    pageNumber = currentPage,
-    rows = rowsPerPage,
+    pageNumber?: number,
+    rows?: number,
   ) => {
-    if (field === sortField) {
-      setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc');
-    } else {
-      setSortOrder('desc');
-      setSortField(field);
+    console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`);
+    const page = pageNumber ?? currentPage;
+    const rowsPerPg = rows ?? rowsPerPage;
+    if (field !== undefined) {
+      if (field === sortField) {
+        // Toggle sort order
+        setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc');
+      } else {
+        // Change sort field and reset order to 'desc'
+        setSortField(field);
+        setSortOrder('desc');
+      }
     }
-    console.log('refreshDocs', field, sortOrder, pageNumber, rows);
-    getDocs(sortField, sortOrder, pageNumber, rows)
+    getDocs(sortField, sortOrder, page, rowsPerPg, true)
       .then((data) => {
-        dispatch(setSourceDocs(data));
-        setFetchedDocuments(data ? data : []);
+        console.log(data);
+        dispatch(setSourceDocs(data ? data.docs : []));
+        setFetchedDocuments(data ? data.docs : []);
+        setTotalPages(data ? data.totalPages : 0);
+        setTotalDocuments(data ? data.totalDocuments : 0);
       })
       .catch((error) => console.error(error))
       .finally(() => {
@@ -118,7 +114,9 @@ const Documents: React.FC<DocumentsProps> = ({
         setLoading(false);
       });
   };
-
+  useEffect(() => {
+    refreshDocs(sortField, currentPage, rowsPerPage);
+  }, []);
   return (
     <div className="mt-8">
       <div className="flex flex-col relative">
@@ -256,12 +254,12 @@ const Documents: React.FC<DocumentsProps> = ({
         rowsPerPage={rowsPerPage}
         onPageChange={(page) => {
           setCurrentPage(page);
-          refreshDocs(sortField, page, rowsPerPage);
+          refreshDocs(undefined, page, rowsPerPage);
         }}
         onRowsPerPageChange={(rows) => {
           setRowsPerPage(rows);
           setCurrentPage(1);
-          refreshDocs(sortField, 1, rows);
+          refreshDocs(undefined, 1, rows);
         }}
       />
     </div>

From 4429755c096fb1b02113aa3917a6db7ef421d863 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Thu, 7 Nov 2024 23:23:01 -0500
Subject: [PATCH 17/42] add paramter type prevent warning lint

---
 frontend/src/settings/Documents.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index 819a4638e..fc21a8f13 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -69,7 +69,7 @@ const Documents: React.FC<DocumentsProps> = ({
   ];
 
   const refreshDocs = (
-    field: 'date' | 'tokens',
+    field: 'date' | 'tokens' | undefined,
     pageNumber?: number,
     rows?: number,
   ) => {

From bc840900a3df86c6ecf35828f54a4d0d5c623340 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Fri, 8 Nov 2024 02:34:31 -0500
Subject: [PATCH 18/42] change logically to upload file one by one instead of
 send all in one

---
 frontend/src/upload/Upload.tsx | 69 +++++++++++++++++++++++++---------
 1 file changed, 51 insertions(+), 18 deletions(-)

diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx
index 81ce9f2b0..7ffb94d14 100644
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -243,26 +243,59 @@ function Upload({
   const doNothing = () => undefined;
 
   const uploadFile = () => {
-    const formData = new FormData();
-    files.forEach((file) => {
-      formData.append('file', file);
-    });
-    formData.append('name', docName);
-    formData.append('user', 'local');
+    if (files.length === 0) {
+      console.error('No files to upload.');
+      return;
+    }
+
     const apiHost = import.meta.env.VITE_API_HOST;
-    const xhr = new XMLHttpRequest();
-    xhr.upload.addEventListener('progress', (event) => {
-      const progress = +((event.loaded / event.total) * 100).toFixed(2);
-      setProgress({ type: 'UPLOAD', percentage: progress });
-    });
-    xhr.onload = () => {
-      const { task_id } = JSON.parse(xhr.responseText);
-      setTimeoutRef.current = setTimeout(() => {
-        setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id });
-      }, 3000);
+
+    const uploadNextFile = (index: number) => {
+      if (index >= files.length) {
+        console.log('All files have been uploaded.');
+        return;
+      }
+
+      const file = files[index];
+      const formData = new FormData();
+      formData.append('file', file);
+      formData.append('name', docName);
+      formData.append('user', 'local');
+
+      const xhr = new XMLHttpRequest();
+      xhr.upload.addEventListener('progress', (event) => {
+        const progress = +((event.loaded / event.total) * 100).toFixed(2);
+        console.log(`File ${index + 1}/${files.length} progress: ${progress}%`);
+        setProgress({ type: 'UPLOAD', percentage: progress });
+      });
+
+      xhr.onload = () => {
+        if (xhr.status >= 200 && xhr.status < 300) {
+          const { task_id } = JSON.parse(xhr.responseText);
+          console.log(`
+            File ${index + 1}/${files.length} uploaded successfully. Task ID: ${task_id}`);
+          setTimeoutRef.current = setTimeout(() => {
+            setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id });
+          }, 3000);
+          uploadNextFile(index + 1);
+        } else {
+          console.error(
+            `
+            File ${index + 1}/${files.length} upload failed with status ${xhr.status}. Response:`,
+            xhr.responseText,
+          );
+        }
+      };
+
+      xhr.onerror = () => {
+        console.error(`
+          File ${index + 1}/${files.length} upload encountered an error.`);
+      };
+
+      xhr.open('POST', `${apiHost + '/api/upload'}`);
+      xhr.send(formData);
     };
-    xhr.open('POST', `${apiHost + '/api/upload'}`);
-    xhr.send(formData);
+    uploadNextFile(0);
   };
 
   const uploadRemote = () => {

From dd9589b37a2f78b7eedf62fe356193fcb3054278 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Fri, 8 Nov 2024 11:02:13 -0500
Subject: [PATCH 19/42] fixing type error

---
 frontend/src/Navigation.tsx              |  5 ++++-
 frontend/src/hooks/useDefaultDocument.ts | 11 +++++-----
 frontend/src/models/misc.ts              |  1 -
 frontend/src/settings/Documents.tsx      | 12 ++++-------
 frontend/src/settings/index.tsx          |  7 +-----
 frontend/src/upload/Upload.tsx           | 27 ++++++++++++++++--------
 6 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx
index 3591469b1..72c0d57a5 100644
--- a/frontend/src/Navigation.tsx
+++ b/frontend/src/Navigation.tsx
@@ -145,7 +145,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         dispatch(setSourceDocs(updatedDocs));
         dispatch(
           setSelectedDocs(
-            updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'),
+            Array.isArray(updatedDocs) &&
+              updatedDocs?.find(
+                (doc: Doc) => doc.name.toLowerCase() === 'default',
+              ),
           ),
         );
       })
diff --git a/frontend/src/hooks/useDefaultDocument.ts b/frontend/src/hooks/useDefaultDocument.ts
index 37374ce0b..7f4b98126 100644
--- a/frontend/src/hooks/useDefaultDocument.ts
+++ b/frontend/src/hooks/useDefaultDocument.ts
@@ -17,11 +17,12 @@ export default function useDefaultDocument() {
     getDocs().then((data) => {
       dispatch(setSourceDocs(data));
       if (!selectedDoc)
-        data?.forEach((doc: Doc) => {
-          if (doc.model && doc.name === 'default') {
-            dispatch(setSelectedDocs(doc));
-          }
-        });
+        Array.isArray(data) &&
+          data?.forEach((doc: Doc) => {
+            if (doc.model && doc.name === 'default') {
+              dispatch(setSelectedDocs(doc));
+            }
+          });
     });
   };
 
diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts
index 5478722c5..3d5d908c5 100644
--- a/frontend/src/models/misc.ts
+++ b/frontend/src/models/misc.ts
@@ -28,7 +28,6 @@ export type PromptProps = {
 };
 
 export type DocumentsProps = {
-  documents: Doc[] | null;
   handleDeleteDocument: (index: number, document: Doc) => void;
 };
 
diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index fc21a8f13..ed7f2d890 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -33,13 +33,9 @@ const formatTokens = (tokens: number): string => {
   }
 };
 
-const Documents: React.FC<DocumentsProps> = ({
-  documents,
-  handleDeleteDocument,
-}) => {
+const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
   const { t } = useTranslation();
   const dispatch = useDispatch();
-
   // State for search input
   const [searchTerm, setSearchTerm] = useState('');
   // State for modal: active/inactive
@@ -73,7 +69,6 @@ const Documents: React.FC<DocumentsProps> = ({
     pageNumber?: number,
     rows?: number,
   ) => {
-    console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`);
     const page = pageNumber ?? currentPage;
     const rowsPerPg = rows ?? rowsPerPage;
     if (field !== undefined) {
@@ -88,7 +83,6 @@ const Documents: React.FC<DocumentsProps> = ({
     }
     getDocs(sortField, sortOrder, page, rowsPerPg, true)
       .then((data) => {
-        console.log(data);
         dispatch(setSourceDocs(data ? data.docs : []));
         setFetchedDocuments(data ? data.docs : []);
         setTotalPages(data ? data.totalPages : 0);
@@ -114,9 +108,11 @@ const Documents: React.FC<DocumentsProps> = ({
         setLoading(false);
       });
   };
+
   useEffect(() => {
     refreshDocs(sortField, currentPage, rowsPerPage);
   }, []);
+
   return (
     <div className="mt-8">
       <div className="flex flex-col relative">
@@ -267,7 +263,7 @@ const Documents: React.FC<DocumentsProps> = ({
 };
 
 Documents.propTypes = {
-  documents: PropTypes.array.isRequired,
+  //documents: PropTypes.array.isRequired,
   handleDeleteDocument: PropTypes.func.isRequired,
 };
 
diff --git a/frontend/src/settings/index.tsx b/frontend/src/settings/index.tsx
index ea3d44281..24ea26812 100644
--- a/frontend/src/settings/index.tsx
+++ b/frontend/src/settings/index.tsx
@@ -70,12 +70,7 @@ export default function Settings() {
       case t('settings.general.label'):
         return <General />;
       case t('settings.documents.label'):
-        return (
-          <Documents
-            documents={documents}
-            handleDeleteDocument={handleDeleteClick}
-          />
-        );
+        return <Documents handleDeleteDocument={handleDeleteClick} />;
       case 'Widgets':
         return (
           <Widgets
diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx
index 81ce9f2b0..443127c05 100644
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -166,7 +166,10 @@ function Upload({
                     dispatch(setSourceDocs(data));
                     dispatch(
                       setSelectedDocs(
-                        data?.find((d) => d.type?.toLowerCase() === 'local'),
+                        Array.isArray(data) &&
+                          data?.find(
+                            (d: Doc) => d.type?.toLowerCase() === 'local',
+                          ),
                       ),
                     );
                   });
@@ -182,15 +185,21 @@ function Upload({
                   getDocs().then((data) => {
                     dispatch(setSourceDocs(data));
                     const docIds = new Set(
-                      sourceDocs?.map((doc: Doc) => (doc.id ? doc.id : null)),
+                      (Array.isArray(sourceDocs) &&
+                        sourceDocs?.map((doc: Doc) =>
+                          doc.id ? doc.id : null,
+                        )) ||
+                        [],
                     );
-                    data?.map((updatedDoc: Doc) => {
-                      if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
-                        //select the doc not present in the intersection of current Docs and fetched data
-                        dispatch(setSelectedDocs(updatedDoc));
-                        return;
-                      }
-                    });
+                    if (data && Array.isArray(data.docs)) {
+                      data.docs.map((updatedDoc: Doc) => {
+                        if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
+                          // Select the doc not present in the intersection of current Docs and fetched data
+                          dispatch(setSelectedDocs(updatedDoc));
+                          return;
+                        }
+                      });
+                    }
                   });
                   setProgress(
                     (progress) =>

From 5debb482651b11e77d3ecf81696b5aedf43cd00b Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Fri, 8 Nov 2024 11:02:13 -0500
Subject: [PATCH 20/42] Paginated With MongoDB / Create New Endpoint change
 routes /combine name, add route /api/source/paginated add new endpoint
 source/paginated fixing table responsive create new function to handling
 api/source/paginated

---
 application/api/user/routes.py            | 96 +++++++++++++++++------
 frontend/src/Navigation.tsx               |  5 +-
 frontend/src/api/endpoints.ts             |  3 +-
 frontend/src/api/services/userService.ts  | 13 +--
 frontend/src/hooks/useDefaultDocument.ts  | 11 +--
 frontend/src/index.css                    |  4 +-
 frontend/src/models/misc.ts               |  2 +-
 frontend/src/preferences/preferenceApi.ts | 78 +++++++++---------
 frontend/src/settings/Documents.tsx       | 40 ++++++----
 frontend/src/settings/index.tsx           |  7 +-
 frontend/src/upload/Upload.tsx            | 27 ++++---
 11 files changed, 174 insertions(+), 112 deletions(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index e5ad68aef..7d44ce9bf 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -7,7 +7,7 @@
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
-from flask import Blueprint, jsonify, make_response, request
+from flask import Blueprint, jsonify, make_response, request, redirect
 from flask_restx import inputs, fields, Namespace, Resource
 from werkzeug.utils import secure_filename
 
@@ -430,14 +430,63 @@ def get(self):
 
 
 @user_ns.route("/api/combine")
+class RedirectToSources(Resource):
+    @api.doc(
+        description="Redirects /api/combine to /api/sources for backward compatibility"
+    )
+    def get(self):
+        return redirect("/api/sources", code=301)
+
+
+@user_ns.route("/api/sources/paginated")
+class PaginatedSources(Resource):
+    @api.doc(description="Get document with pagination, sorting and filtering")
+    def get(self):
+        user = "local"
+        sort_field = request.args.get("sort", "date")  # Default to 'date'
+        sort_order = request.args.get("order", "desc")  # Default to 'desc'
+        page = int(request.args.get("page", 1))  # Default to 1
+        rows_per_page = int(request.args.get("rows", 10))  # Default to 10
+
+        # Prepare
+        query = {"user": user}
+        total_documents = sources_collection.count_documents(query)
+        total_pages = max(1, math.ceil(total_documents / rows_per_page))
+        sort_order = 1 if sort_order == "asc" else -1
+        skip = (page - 1) * rows_per_page
+
+        try:
+            documents = (
+                sources_collection.find(query)
+                .sort(sort_field, sort_order)
+                .skip(skip)
+                .limit(rows_per_page)
+            )
+
+            paginated_docs = []
+            for doc in documents:
+                doc["_id"] = str(doc["_id"])
+                paginated_docs.append(doc)
+
+            response = {
+                "total": total_documents,
+                "totalPages": total_pages,
+                "currentPage": page,
+                "paginated": paginated_docs,
+            }
+            return make_response(jsonify(response), 200)
+
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+
+@user_ns.route("/api/sources")
 class CombinedJson(Resource):
     @api.doc(description="Provide JSON file with combined available indexes")
     def get(self):
         user = "local"
-        sort_field = request.args.get('sort', 'date')  # Default to 'date'
-        sort_order = request.args.get('order', "desc")  # Default to 'desc'
-        page = int(request.args.get('page', 1)) # Default to 1
-        rows_per_page = int(request.args.get('rows', 10)) # Default to 10
+        sort_field = request.args.get("sort", "date")  # Default to 'date'
+        sort_order = request.args.get("order", "desc")  # Default to 'desc'
         data = [
             {
                 "name": "default",
@@ -450,7 +499,9 @@ def get(self):
         ]
 
         try:
-            for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1):
+            for index in sources_collection.find({"user": user}).sort(
+                sort_field, 1 if sort_order == "asc" else -1
+            ):
                 data.append(
                     {
                         "id": str(index["_id"]),
@@ -488,23 +539,11 @@ def get(self):
                         "retriever": "brave_search",
                     }
                 )
-            total_documents = len(data)
-            total_pages = max(1, math.ceil(total_documents / rows_per_page))
-
-            first_index = (page - 1) * rows_per_page
-            last_index = first_index + rows_per_page
-            paginated_docs = data[first_index:last_index]
 
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
-            
-        response = {
-            "paginated_docs": paginated_docs,
-            "totalDocuments": total_documents,
-            "totalPages": total_pages,
-            "documents":data
-        }
-        return make_response(jsonify(response), 200)
+
+        return make_response(jsonify(data), 200)
 
 
 @user_ns.route("/api/docs_check")
@@ -1690,7 +1729,9 @@ class TextToSpeech(Resource):
     tts_model = api.model(
         "TextToSpeechModel",
         {
-            "text": fields.String(required=True, description="Text to be synthesized as audio"),
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
         },
     )
 
@@ -1702,8 +1743,15 @@ def post(self):
         try:
             tts_instance = GoogleTTS()
             audio_base64, detected_language = tts_instance.text_to_speech(text)
-            return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
-
-
diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx
index 3591469b1..72c0d57a5 100644
--- a/frontend/src/Navigation.tsx
+++ b/frontend/src/Navigation.tsx
@@ -145,7 +145,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
         dispatch(setSourceDocs(updatedDocs));
         dispatch(
           setSelectedDocs(
-            updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'),
+            Array.isArray(updatedDocs) &&
+              updatedDocs?.find(
+                (doc: Doc) => doc.name.toLowerCase() === 'default',
+              ),
           ),
         );
       })
diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts
index 84674049e..4e7112d04 100644
--- a/frontend/src/api/endpoints.ts
+++ b/frontend/src/api/endpoints.ts
@@ -1,7 +1,8 @@
 const endpoints = {
   USER: {
-    DOCS: '/api/combine',
+    DOCS: '/api/sources',
     DOCS_CHECK: '/api/docs_check',
+    DOCS_PAGINATED: '/api/sources/paginated',
     API_KEYS: '/api/get_api_keys',
     CREATE_API_KEY: '/api/create_api_key',
     DELETE_API_KEY: '/api/delete_api_key',
diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts
index bebadca50..82f322449 100644
--- a/frontend/src/api/services/userService.ts
+++ b/frontend/src/api/services/userService.ts
@@ -2,15 +2,10 @@ import apiClient from '../client';
 import endpoints from '../endpoints';
 
 const userService = {
-  getDocs: (
-    sort: string,
-    order: string,
-    pageNumber: number,
-    rowsPerPage: number,
-  ): Promise<any> =>
-    apiClient.get(
-      `${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`,
-    ),
+  getDocs: (sort: string, order: string): Promise<any> =>
+    apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`),
+  getDocsWithPagination: (query: string): Promise<any> =>
+    apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`),
   checkDocs: (data: any): Promise<any> =>
     apiClient.post(endpoints.USER.DOCS_CHECK, data),
   getAPIKeys: (): Promise<any> => apiClient.get(endpoints.USER.API_KEYS),
diff --git a/frontend/src/hooks/useDefaultDocument.ts b/frontend/src/hooks/useDefaultDocument.ts
index 37374ce0b..7f4b98126 100644
--- a/frontend/src/hooks/useDefaultDocument.ts
+++ b/frontend/src/hooks/useDefaultDocument.ts
@@ -17,11 +17,12 @@ export default function useDefaultDocument() {
     getDocs().then((data) => {
       dispatch(setSourceDocs(data));
       if (!selectedDoc)
-        data?.forEach((doc: Doc) => {
-          if (doc.model && doc.name === 'default') {
-            dispatch(setSelectedDocs(doc));
-          }
-        });
+        Array.isArray(data) &&
+          data?.forEach((doc: Doc) => {
+            if (doc.model && doc.name === 'default') {
+              dispatch(setSelectedDocs(doc));
+            }
+          });
     });
   };
 
diff --git a/frontend/src/index.css b/frontend/src/index.css
index 090a31ca6..e8e9c9bdc 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -50,11 +50,11 @@ body.dark {
 
 @layer components {
   .table-default {
-    @apply block w-max mx-auto table-fixed content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
+    @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
   }
 
   .table-default th {
-    @apply p-4 w-[250px] font-normal text-gray-400; /* Remove border-r */
+    @apply p-4 w-1/4 font-normal text-gray-400 text-nowrap; /* Remove border-r */
   }
 
   .table-default th:last-child {
diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts
index 5478722c5..46af487c7 100644
--- a/frontend/src/models/misc.ts
+++ b/frontend/src/models/misc.ts
@@ -18,6 +18,7 @@ export type GetDocsResponse = {
   docs: Doc[];
   totalDocuments: number;
   totalPages: number;
+  nextCursor: string;
 };
 
 export type PromptProps = {
@@ -28,7 +29,6 @@ export type PromptProps = {
 };
 
 export type DocumentsProps = {
-  documents: Doc[] | null;
   handleDeleteDocument: (index: number, document: Doc) => void;
 };
 
diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts
index e77c9e3b2..e3d22c02a 100644
--- a/frontend/src/preferences/preferenceApi.ts
+++ b/frontend/src/preferences/preferenceApi.ts
@@ -4,49 +4,53 @@ import { Doc, GetDocsResponse } from '../models/misc';
 
 //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
 export async function getDocs(
-  sort?: string,
-  order?: string,
-  pageNumber?: number,
-  rowsPerPage?: number,
-  withPagination?: true,
-): Promise<GetDocsResponse | null>;
+  sort = 'date',
+  order = 'desc',
+): Promise<Doc[] | null> {
+  try {
+    const response = await userService.getDocs(sort, order);
+    const data = await response.json();
 
-export async function getDocs(
+    const docs: Doc[] = [];
+    console.log(data);
+    data.forEach((doc: object) => {
+      docs.push(doc as Doc);
+    });
+
+    return docs;
+  } catch (error) {
+    console.log(error);
+    return null;
+  }
+}
+
+export async function getDocsWithPagination(
   sort = 'date',
   order = 'desc',
   pageNumber = 1,
-  rowsPerPage = 5,
-  withPagination = false,
-): Promise<Doc[] | GetDocsResponse | null> {
+  rowsPerPage = 10,
+): Promise<GetDocsResponse | null> {
   try {
-    const response = await userService.getDocs(
-      sort,
-      order,
-      pageNumber,
-      rowsPerPage,
-    );
+    const query = `sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`;
+    const response = await userService.getDocsWithPagination(query);
     const data = await response.json();
-    console.log(data);
-    if (withPagination) {
-      const docs: Doc[] = [];
-      Array.isArray(data.paginated_docs) &&
-        data.paginated_docs.forEach((doc: object) => {
-          docs.push(doc as Doc);
-        });
-
-      const totalDocuments = data.totalDocuments || 0;
-      const totalPages = data.totalPages || 0;
-      console.log(`totalDocuments: ${totalDocuments}`);
-      console.log(`totalPages: ${totalPages}`);
-      return { docs, totalDocuments, totalPages };
-    } else {
-      const docs: Doc[] = [];
-      Array.isArray(data.documents) &&
-        data.documents.forEach((doc: object) => {
-          docs.push(doc as Doc);
-        });
-      return docs;
-    }
+
+    const docs: Doc[] = [];
+    console.log(`data: ${data}`);
+    Array.isArray(data.paginated) &&
+      data.paginated.forEach((doc: Doc) => {
+        docs.push(doc as Doc);
+      });
+    console.log(`total: ${data.total}`);
+    console.log(`totalPages: ${data.totalPages}`);
+    console.log(`cursor: ${data.nextCursor}`);
+    console.log(`currentPage: ${data.currentPage}`);
+    return {
+      docs: docs,
+      totalDocuments: data.total,
+      totalPages: data.totalPages,
+      nextCursor: data.nextCursor,
+    };
   } catch (error) {
     console.log(error);
     return null;
diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index fc21a8f13..8a35ca7d8 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -10,7 +10,7 @@ import caretSort from '../assets/caret-sort.svg';
 import DropdownMenu from '../components/DropdownMenu';
 import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported
 import SkeletonLoader from '../components/SkeletonLoader';
-import { getDocs } from '../preferences/preferenceApi';
+import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi';
 import { setSourceDocs } from '../preferences/preferenceSlice';
 import Input from '../components/Input';
 import Upload from '../upload/Upload'; // Import the Upload component
@@ -33,13 +33,9 @@ const formatTokens = (tokens: number): string => {
   }
 };
 
-const Documents: React.FC<DocumentsProps> = ({
-  documents,
-  handleDeleteDocument,
-}) => {
+const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
   const { t } = useTranslation();
   const dispatch = useDispatch();
-
   // State for search input
   const [searchTerm, setSearchTerm] = useState('');
   // State for modal: active/inactive
@@ -60,7 +56,6 @@ const Documents: React.FC<DocumentsProps> = ({
   );
   // State for documents
   const currentDocuments = filteredDocuments ?? [];
-
   const syncOptions = [
     { label: 'Never', value: 'never' },
     { label: 'Daily', value: 'daily' },
@@ -73,9 +68,9 @@ const Documents: React.FC<DocumentsProps> = ({
     pageNumber?: number,
     rows?: number,
   ) => {
-    console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`);
     const page = pageNumber ?? currentPage;
     const rowsPerPg = rows ?? rowsPerPage;
+
     if (field !== undefined) {
       if (field === sortField) {
         // Toggle sort order
@@ -86,9 +81,9 @@ const Documents: React.FC<DocumentsProps> = ({
         setSortOrder('desc');
       }
     }
-    getDocs(sortField, sortOrder, page, rowsPerPg, true)
+    getDocsWithPagination(sortField, sortOrder, page, rowsPerPg)
       .then((data) => {
-        console.log(data);
+        console.log('Data received from getDocsWithPagination:', data);
         dispatch(setSourceDocs(data ? data.docs : []));
         setFetchedDocuments(data ? data.docs : []);
         setTotalPages(data ? data.totalPages : 0);
@@ -99,6 +94,7 @@ const Documents: React.FC<DocumentsProps> = ({
         setLoading(false);
       });
   };
+
   const handleManageSync = (doc: Doc, sync_frequency: string) => {
     setLoading(true);
     userService
@@ -114,9 +110,13 @@ const Documents: React.FC<DocumentsProps> = ({
         setLoading(false);
       });
   };
+
   useEffect(() => {
-    refreshDocs(sortField, currentPage, rowsPerPage);
-  }, []);
+    if (modalState === 'INACTIVE') {
+      refreshDocs(sortField, currentPage, rowsPerPage);
+    }
+  }, [modalState, sortField, currentPage, rowsPerPage]);
+
   return (
     <div className="mt-8">
       <div className="flex flex-col relative">
@@ -190,7 +190,7 @@ const Documents: React.FC<DocumentsProps> = ({
                 )}
                 {Array.isArray(currentDocuments) &&
                   currentDocuments.map((document, index) => (
-                    <tr key={index}>
+                    <tr key={index} className="text-nowrap font-normal">
                       <td>{document.name}</td>
                       <td>{document.date}</td>
                       <td>
@@ -248,18 +248,24 @@ const Documents: React.FC<DocumentsProps> = ({
           </div>
         )}
       </div>
+      {/* Pagination component with props:
+      # Note: Every time the page changes, 
+      the refreshDocs function is called with the updated page number and rows per page.
+      and reset cursor paginated query parameter to undefined.
+      */}
       <Pagination
         currentPage={currentPage}
         totalPages={totalPages}
         rowsPerPage={rowsPerPage}
         onPageChange={(page) => {
           setCurrentPage(page);
-          refreshDocs(undefined, page, rowsPerPage);
+          refreshDocs(sortField, page, rowsPerPage); // Pass `true` to reset lastID if not using cursor
         }}
         onRowsPerPageChange={(rows) => {
+          console.log('Pagination - Rows per Page Change:', rows);
           setRowsPerPage(rows);
-          setCurrentPage(1);
-          refreshDocs(undefined, 1, rows);
+          setCurrentPage(1); // Reset to page 1 on rows per page change
+          refreshDocs(sortField, 1, rows); // Reset lastID for fresh pagination
         }}
       />
     </div>
@@ -267,7 +273,7 @@ const Documents: React.FC<DocumentsProps> = ({
 };
 
 Documents.propTypes = {
-  documents: PropTypes.array.isRequired,
+  //documents: PropTypes.array.isRequired,
   handleDeleteDocument: PropTypes.func.isRequired,
 };
 
diff --git a/frontend/src/settings/index.tsx b/frontend/src/settings/index.tsx
index ea3d44281..24ea26812 100644
--- a/frontend/src/settings/index.tsx
+++ b/frontend/src/settings/index.tsx
@@ -70,12 +70,7 @@ export default function Settings() {
       case t('settings.general.label'):
         return <General />;
       case t('settings.documents.label'):
-        return (
-          <Documents
-            documents={documents}
-            handleDeleteDocument={handleDeleteClick}
-          />
-        );
+        return <Documents handleDeleteDocument={handleDeleteClick} />;
       case 'Widgets':
         return (
           <Widgets
diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx
index 81ce9f2b0..443127c05 100644
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -166,7 +166,10 @@ function Upload({
                     dispatch(setSourceDocs(data));
                     dispatch(
                       setSelectedDocs(
-                        data?.find((d) => d.type?.toLowerCase() === 'local'),
+                        Array.isArray(data) &&
+                          data?.find(
+                            (d: Doc) => d.type?.toLowerCase() === 'local',
+                          ),
                       ),
                     );
                   });
@@ -182,15 +185,21 @@ function Upload({
                   getDocs().then((data) => {
                     dispatch(setSourceDocs(data));
                     const docIds = new Set(
-                      sourceDocs?.map((doc: Doc) => (doc.id ? doc.id : null)),
+                      (Array.isArray(sourceDocs) &&
+                        sourceDocs?.map((doc: Doc) =>
+                          doc.id ? doc.id : null,
+                        )) ||
+                        [],
                     );
-                    data?.map((updatedDoc: Doc) => {
-                      if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
-                        //select the doc not present in the intersection of current Docs and fetched data
-                        dispatch(setSelectedDocs(updatedDoc));
-                        return;
-                      }
-                    });
+                    if (data && Array.isArray(data.docs)) {
+                      data.docs.map((updatedDoc: Doc) => {
+                        if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
+                          // Select the doc not present in the intersection of current Docs and fetched data
+                          dispatch(setSelectedDocs(updatedDoc));
+                          return;
+                        }
+                      });
+                    }
                   });
                   setProgress(
                     (progress) =>

From d59ffaf0bd6a3612331fc0d0cf960681851897c5 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sat, 9 Nov 2024 14:57:15 -0500
Subject: [PATCH 21/42] fix sorting on /sources

---
 application/api/user/routes.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index 7d44ce9bf..b3062005c 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -485,8 +485,6 @@ class CombinedJson(Resource):
     @api.doc(description="Provide JSON file with combined available indexes")
     def get(self):
         user = "local"
-        sort_field = request.args.get("sort", "date")  # Default to 'date'
-        sort_order = request.args.get("order", "desc")  # Default to 'desc'
         data = [
             {
                 "name": "default",
@@ -500,7 +498,7 @@ def get(self):
 
         try:
             for index in sources_collection.find({"user": user}).sort(
-                sort_field, 1 if sort_order == "asc" else -1
+                "date", -1
             ):
                 data.append(
                     {

From f3a005a66753149cc80e5adc95bdc7cfe0975b6a Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sat, 9 Nov 2024 14:57:30 -0500
Subject: [PATCH 22/42] remove params on getDocs

---
 frontend/src/api/services/userService.ts  | 3 +--
 frontend/src/preferences/preferenceApi.ts | 7 ++-----
 frontend/src/settings/Documents.tsx       | 3 +--
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts
index 82f322449..942318ae4 100644
--- a/frontend/src/api/services/userService.ts
+++ b/frontend/src/api/services/userService.ts
@@ -2,8 +2,7 @@ import apiClient from '../client';
 import endpoints from '../endpoints';
 
 const userService = {
-  getDocs: (sort: string, order: string): Promise<any> =>
-    apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`),
+  getDocs: (): Promise<any> => apiClient.get(`${endpoints.USER.DOCS}`),
   getDocsWithPagination: (query: string): Promise<any> =>
     apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`),
   checkDocs: (data: any): Promise<any> =>
diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts
index e3d22c02a..c1ac97de7 100644
--- a/frontend/src/preferences/preferenceApi.ts
+++ b/frontend/src/preferences/preferenceApi.ts
@@ -3,12 +3,9 @@ import userService from '../api/services/userService';
 import { Doc, GetDocsResponse } from '../models/misc';
 
 //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
-export async function getDocs(
-  sort = 'date',
-  order = 'desc',
-): Promise<Doc[] | null> {
+export async function getDocs(): Promise<Doc[] | null> {
   try {
-    const response = await userService.getDocs(sort, order);
+    const response = await userService.getDocs();
     const data = await response.json();
 
     const docs: Doc[] = [];
diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index 8a35ca7d8..70cc86bab 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -83,8 +83,7 @@ const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
     }
     getDocsWithPagination(sortField, sortOrder, page, rowsPerPg)
       .then((data) => {
-        console.log('Data received from getDocsWithPagination:', data);
-        dispatch(setSourceDocs(data ? data.docs : []));
+        //dispatch(setSourceDocs(data ? data.docs : []));
         setFetchedDocuments(data ? data.docs : []);
         setTotalPages(data ? data.totalPages : 0);
         setTotalDocuments(data ? data.totalDocuments : 0);

From 0837295bd3fa6d7fdc5b225c8b4c1d25e8271991 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sat, 9 Nov 2024 16:24:21 -0500
Subject: [PATCH 23/42] fix table responsive issue

---
 frontend/src/index.css | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/src/index.css b/frontend/src/index.css
index e8e9c9bdc..1c72035e8 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -50,11 +50,11 @@ body.dark {
 
 @layer components {
   .table-default {
-    @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
+    @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray overflow-auto;
   }
 
   .table-default th {
-    @apply p-4 w-1/4 font-normal text-gray-400 text-nowrap; /* Remove border-r */
+    @apply p-4 w-1/3 font-normal text-gray-400 text-nowrap; /* Remove border-r */
   }
 
   .table-default th:last-child {

From e00c6f2c140962f0007e2a16c6b2fd4408192d68 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sat, 9 Nov 2024 16:59:00 -0500
Subject: [PATCH 24/42] add min width for delete button, dropdown dakrtheme

---
 frontend/src/components/DocumentPagination.tsx | 2 +-
 frontend/src/index.css                         | 2 +-
 frontend/src/settings/Documents.tsx            | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/frontend/src/components/DocumentPagination.tsx b/frontend/src/components/DocumentPagination.tsx
index b5d82456b..3edeff9df 100644
--- a/frontend/src/components/DocumentPagination.tsx
+++ b/frontend/src/components/DocumentPagination.tsx
@@ -48,7 +48,7 @@ const Pagination: React.FC<PaginationProps> = ({
         <select
           value={rowsPerPage}
           onChange={(e) => onRowsPerPageChange(Number(e.target.value))}
-          className="border border-gray-300 rounded px-2 py-1"
+          className="border border-gray-300 rounded px-2 py-1 dark:bg-transparent dark:text-gray-50"
         >
           {rowsPerPageOptions.map((option) => (
             <option key={option} value={option}>
diff --git a/frontend/src/index.css b/frontend/src/index.css
index 1c72035e8..6de42c244 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -50,7 +50,7 @@ body.dark {
 
 @layer components {
   .table-default {
-    @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray overflow-auto;
+    @apply block w-full mx-auto table-auto content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray overflow-auto;
   }
 
   .table-default th {
diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index 70cc86bab..26af09d5d 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -199,7 +199,7 @@ const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
                         {document.type === 'remote' ? 'Pre-loaded' : 'Private'}
                       </td>
                       <td>
-                        <div className="flex flex-row items-center">
+                        <div className="min-w-[70px] flex flex-row items-end justify-end ml-auto">
                           {document.type !== 'remote' && (
                             <img
                               src={Trash}

From 8a225e279fcfa57d2205cecf763a1a46a967db4f Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sun, 10 Nov 2024 02:59:09 -0500
Subject: [PATCH 25/42] multiple-file missing task ingest

restore upload to pack all file together instead one by one
---
 application/api/user/routes.py | 55 ++++++++++++++++++++++-----
 frontend/src/upload/Upload.tsx | 69 +++++++++-------------------------
 2 files changed, 64 insertions(+), 60 deletions(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index 8e62683e6..0868333ee 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -315,7 +315,7 @@ def post(self):
                 for file in files:
                     filename = secure_filename(file.filename)
                     file.save(os.path.join(temp_dir, filename))
-
+                    print(f"Saved file: {filename}")
                 zip_path = shutil.make_archive(
                     base_name=os.path.join(save_dir, job_name),
                     format="zip",
@@ -323,6 +323,26 @@ def post(self):
                 )
                 final_filename = os.path.basename(zip_path)
                 shutil.rmtree(temp_dir)
+                task = ingest.delay(
+                    settings.UPLOAD_FOLDER,
+                    [
+                        ".rst",
+                        ".md",
+                        ".pdf",
+                        ".txt",
+                        ".docx",
+                        ".csv",
+                        ".epub",
+                        ".html",
+                        ".mdx",
+                        ".json",
+                        ".xlsx",
+                        ".pptx",
+                    ],
+                    job_name,
+                    final_filename,
+                    user,
+                )
             else:
                 file = files[0]
                 final_filename = secure_filename(file.filename)
@@ -349,9 +369,10 @@ def post(self):
                     final_filename,
                     user,
                 )
+
         except Exception as err:
+            print(f"Error: {err}")
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
-
         return make_response(jsonify({"success": True, "task_id": task.id}), 200)
 
 
@@ -422,6 +443,11 @@ def get(self):
 
             task = celery.AsyncResult(task_id)
             task_meta = task.info
+            print(f"Task status: {task.status}")
+            if not isinstance(
+                task_meta, (dict, list, str, int, float, bool, type(None))
+            ):
+                task_meta = str(task_meta)  # Convert to a string representation
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
@@ -433,8 +459,8 @@ class CombinedJson(Resource):
     @api.doc(description="Provide JSON file with combined available indexes")
     def get(self):
         user = "local"
-        sort_field = request.args.get('sort', 'date')  # Default to 'date'
-        sort_order = request.args.get('order', "desc")  # Default to 'desc'
+        sort_field = request.args.get("sort", "date")  # Default to 'date'
+        sort_order = request.args.get("order", "desc")  # Default to 'desc'
         data = [
             {
                 "name": "default",
@@ -447,7 +473,9 @@ def get(self):
         ]
 
         try:
-            for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1):
+            for index in sources_collection.find({"user": user}).sort(
+                sort_field, 1 if sort_order == "asc" else -1
+            ):
                 data.append(
                     {
                         "id": str(index["_id"]),
@@ -1674,7 +1702,9 @@ class TextToSpeech(Resource):
     tts_model = api.model(
         "TextToSpeechModel",
         {
-            "text": fields.String(required=True, description="Text to be synthesized as audio"),
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
         },
     )
 
@@ -1686,8 +1716,15 @@ def post(self):
         try:
             tts_instance = GoogleTTS()
             audio_base64, detected_language = tts_instance.text_to_speech(text)
-            return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
-
-
diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx
index 7ffb94d14..81ce9f2b0 100644
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -243,59 +243,26 @@ function Upload({
   const doNothing = () => undefined;
 
   const uploadFile = () => {
-    if (files.length === 0) {
-      console.error('No files to upload.');
-      return;
-    }
-
-    const apiHost = import.meta.env.VITE_API_HOST;
-
-    const uploadNextFile = (index: number) => {
-      if (index >= files.length) {
-        console.log('All files have been uploaded.');
-        return;
-      }
-
-      const file = files[index];
-      const formData = new FormData();
+    const formData = new FormData();
+    files.forEach((file) => {
       formData.append('file', file);
-      formData.append('name', docName);
-      formData.append('user', 'local');
-
-      const xhr = new XMLHttpRequest();
-      xhr.upload.addEventListener('progress', (event) => {
-        const progress = +((event.loaded / event.total) * 100).toFixed(2);
-        console.log(`File ${index + 1}/${files.length} progress: ${progress}%`);
-        setProgress({ type: 'UPLOAD', percentage: progress });
-      });
-
-      xhr.onload = () => {
-        if (xhr.status >= 200 && xhr.status < 300) {
-          const { task_id } = JSON.parse(xhr.responseText);
-          console.log(`
-            File ${index + 1}/${files.length} uploaded successfully. Task ID: ${task_id}`);
-          setTimeoutRef.current = setTimeout(() => {
-            setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id });
-          }, 3000);
-          uploadNextFile(index + 1);
-        } else {
-          console.error(
-            `
-            File ${index + 1}/${files.length} upload failed with status ${xhr.status}. Response:`,
-            xhr.responseText,
-          );
-        }
-      };
-
-      xhr.onerror = () => {
-        console.error(`
-          File ${index + 1}/${files.length} upload encountered an error.`);
-      };
-
-      xhr.open('POST', `${apiHost + '/api/upload'}`);
-      xhr.send(formData);
+    });
+    formData.append('name', docName);
+    formData.append('user', 'local');
+    const apiHost = import.meta.env.VITE_API_HOST;
+    const xhr = new XMLHttpRequest();
+    xhr.upload.addEventListener('progress', (event) => {
+      const progress = +((event.loaded / event.total) * 100).toFixed(2);
+      setProgress({ type: 'UPLOAD', percentage: progress });
+    });
+    xhr.onload = () => {
+      const { task_id } = JSON.parse(xhr.responseText);
+      setTimeoutRef.current = setTimeout(() => {
+        setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id });
+      }, 3000);
     };
-    uploadNextFile(0);
+    xhr.open('POST', `${apiHost + '/api/upload'}`);
+    xhr.send(formData);
   };
 
   const uploadRemote = () => {

From b19e9cae236a4cf54abc1dc0c1a6776eda9ddf5a Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sun, 10 Nov 2024 14:38:51 -0500
Subject: [PATCH 26/42] backend fix passing id to paginated docs

---
 application/api/user/routes.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index b3062005c..46ed15cde 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -465,7 +465,9 @@ def get(self):
 
             paginated_docs = []
             for doc in documents:
-                doc["_id"] = str(doc["_id"])
+                print(doc)
+                doc["id"] = str(doc["_id"])
+                del doc["_id"]
                 paginated_docs.append(doc)
 
             response = {

From 839f0a3b95f6d46743f6c43521efc93705160ec8 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sun, 10 Nov 2024 14:39:20 -0500
Subject: [PATCH 27/42] remove log on api, add paginatedDoc on redux

---
 frontend/src/preferences/preferenceApi.ts   | 7 -------
 frontend/src/preferences/preferenceSlice.ts | 8 ++++++++
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts
index c1ac97de7..32cf8b178 100644
--- a/frontend/src/preferences/preferenceApi.ts
+++ b/frontend/src/preferences/preferenceApi.ts
@@ -9,7 +9,6 @@ export async function getDocs(): Promise<Doc[] | null> {
     const data = await response.json();
 
     const docs: Doc[] = [];
-    console.log(data);
     data.forEach((doc: object) => {
       docs.push(doc as Doc);
     });
@@ -31,17 +30,11 @@ export async function getDocsWithPagination(
     const query = `sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`;
     const response = await userService.getDocsWithPagination(query);
     const data = await response.json();
-
     const docs: Doc[] = [];
-    console.log(`data: ${data}`);
     Array.isArray(data.paginated) &&
       data.paginated.forEach((doc: Doc) => {
         docs.push(doc as Doc);
       });
-    console.log(`total: ${data.total}`);
-    console.log(`totalPages: ${data.totalPages}`);
-    console.log(`cursor: ${data.nextCursor}`);
-    console.log(`currentPage: ${data.currentPage}`);
     return {
       docs: docs,
       totalDocuments: data.total,
diff --git a/frontend/src/preferences/preferenceSlice.ts b/frontend/src/preferences/preferenceSlice.ts
index c566ba70b..8b3064d58 100644
--- a/frontend/src/preferences/preferenceSlice.ts
+++ b/frontend/src/preferences/preferenceSlice.ts
@@ -20,6 +20,7 @@ export interface Preference {
     loading: boolean;
   };
   modalState: ActiveState;
+  paginatedDocuments: Doc[] | null;
 }
 
 const initialState: Preference = {
@@ -42,6 +43,7 @@ const initialState: Preference = {
     loading: false,
   },
   modalState: 'INACTIVE',
+  paginatedDocuments: null,
 };
 
 export const prefSlice = createSlice({
@@ -57,6 +59,9 @@ export const prefSlice = createSlice({
     setSourceDocs: (state, action) => {
       state.sourceDocs = action.payload;
     },
+    setPaginatedDocuments: (state, action) => {
+      state.paginatedDocuments = action.payload;
+    },
     setConversations: (state, action) => {
       state.conversations = action.payload;
     },
@@ -84,6 +89,7 @@ export const {
   setChunks,
   setTokenLimit,
   setModalStateDeleteConv,
+  setPaginatedDocuments,
 } = prefSlice.actions;
 export default prefSlice.reducer;
 
@@ -155,3 +161,5 @@ export const selectPrompt = (state: RootState) => state.preference.prompt;
 export const selectChunks = (state: RootState) => state.preference.chunks;
 export const selectTokenLimit = (state: RootState) =>
   state.preference.token_limit;
+export const selectPaginatedDocuments = (state: RootState) =>
+  state.preference.paginatedDocuments;

From 1056c943d3de790cb96ddd829eb0e65c9d576a50 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sun, 10 Nov 2024 14:39:56 -0500
Subject: [PATCH 28/42] fix remove button on navigation sidebar and
 setting/document

---
 frontend/src/Navigation.tsx                   |  9 +++++
 .../src/components/DocumentPagination.tsx     |  8 +++--
 frontend/src/models/misc.ts                   |  1 +
 frontend/src/settings/Documents.tsx           | 33 ++++++++++---------
 frontend/src/settings/index.tsx               | 28 ++++++++++++----
 5 files changed, 55 insertions(+), 24 deletions(-)

diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx
index 72c0d57a5..324d5aa05 100644
--- a/frontend/src/Navigation.tsx
+++ b/frontend/src/Navigation.tsx
@@ -34,10 +34,12 @@ import {
   selectSelectedDocs,
   selectSelectedDocsStatus,
   selectSourceDocs,
+  selectPaginatedDocuments,
   setConversations,
   setModalStateDeleteConv,
   setSelectedDocs,
   setSourceDocs,
+  setPaginatedDocuments,
 } from './preferences/preferenceSlice';
 import Spinner from './assets/spinner.svg';
 import SpinnerDark from './assets/spinner-dark.svg';
@@ -72,6 +74,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   const conversations = useSelector(selectConversations);
   const modalStateDeleteConv = useSelector(selectModalStateDeleteConv);
   const conversationId = useSelector(selectConversationId);
+  const paginatedDocuments = useSelector(selectPaginatedDocuments);
   const [isDeletingConversation, setIsDeletingConversation] = useState(false);
 
   const { isMobile } = useMediaQuery();
@@ -143,6 +146,12 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
       })
       .then((updatedDocs) => {
         dispatch(setSourceDocs(updatedDocs));
+        const updatedPaginatedDocs = paginatedDocuments?.filter(
+          (document) => document.id !== doc.id,
+        );
+        dispatch(
+          setPaginatedDocuments(updatedPaginatedDocs || paginatedDocuments),
+        );
         dispatch(
           setSelectedDocs(
             Array.isArray(updatedDocs) &&
diff --git a/frontend/src/components/DocumentPagination.tsx b/frontend/src/components/DocumentPagination.tsx
index 3edeff9df..b0532362b 100644
--- a/frontend/src/components/DocumentPagination.tsx
+++ b/frontend/src/components/DocumentPagination.tsx
@@ -48,10 +48,14 @@ const Pagination: React.FC<PaginationProps> = ({
         <select
           value={rowsPerPage}
           onChange={(e) => onRowsPerPageChange(Number(e.target.value))}
-          className="border border-gray-300 rounded px-2 py-1 dark:bg-transparent dark:text-gray-50"
+          className="border border-gray-300 rounded px-2 py-1 dark:bg-dark-charcoal dark:text-gray-50"
         >
           {rowsPerPageOptions.map((option) => (
-            <option key={option} value={option}>
+            <option
+              className="bg-white dark:bg-dark-charcoal dark:text-gray-50"
+              key={option}
+              value={option}
+            >
               {option}
             </option>
           ))}
diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts
index 46af487c7..0d9c89314 100644
--- a/frontend/src/models/misc.ts
+++ b/frontend/src/models/misc.ts
@@ -29,6 +29,7 @@ export type PromptProps = {
 };
 
 export type DocumentsProps = {
+  paginatedDocuments: Doc[] | null;
   handleDeleteDocument: (index: number, document: Doc) => void;
 };
 
diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index 26af09d5d..eaee879e5 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -1,20 +1,20 @@
 import React, { useState, useEffect } from 'react';
 import PropTypes from 'prop-types';
-import { useTranslation } from 'react-i18next';
-import { useDispatch } from 'react-redux';
-
 import userService from '../api/services/userService';
 import SyncIcon from '../assets/sync.svg';
 import Trash from '../assets/trash.svg';
 import caretSort from '../assets/caret-sort.svg';
 import DropdownMenu from '../components/DropdownMenu';
-import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported
 import SkeletonLoader from '../components/SkeletonLoader';
-import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi';
-import { setSourceDocs } from '../preferences/preferenceSlice';
 import Input from '../components/Input';
 import Upload from '../upload/Upload'; // Import the Upload component
 import Pagination from '../components/DocumentPagination';
+import { useTranslation } from 'react-i18next';
+import { useDispatch } from 'react-redux';
+import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported
+import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi';
+import { setSourceDocs } from '../preferences/preferenceSlice';
+import { setPaginatedDocuments } from '../preferences/preferenceSlice';
 
 // Utility function to format numbers
 const formatTokens = (tokens: number): string => {
@@ -33,7 +33,10 @@ const formatTokens = (tokens: number): string => {
   }
 };
 
-const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
+const Documents: React.FC<DocumentsProps> = ({
+  paginatedDocuments,
+  handleDeleteDocument,
+}) => {
   const { t } = useTranslation();
   const dispatch = useDispatch();
   // State for search input
@@ -48,10 +51,9 @@ const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
   const [currentPage, setCurrentPage] = useState<number>(1);
   const [rowsPerPage, setRowsPerPage] = useState<number>(10);
   const [totalPages, setTotalPages] = useState<number>(1);
-  const [totalDocuments, setTotalDocuments] = useState<number>(0);
-  const [fetchedDocuments, setFetchedDocuments] = useState<Doc[]>([]);
+  // const [totalDocuments, setTotalDocuments] = useState<number>(0);
   // Filter documents based on the search term
-  const filteredDocuments = fetchedDocuments?.filter((document) =>
+  const filteredDocuments = paginatedDocuments?.filter((document) =>
     document.name.toLowerCase().includes(searchTerm.toLowerCase()),
   );
   // State for documents
@@ -84,9 +86,9 @@ const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
     getDocsWithPagination(sortField, sortOrder, page, rowsPerPg)
       .then((data) => {
         //dispatch(setSourceDocs(data ? data.docs : []));
-        setFetchedDocuments(data ? data.docs : []);
+        dispatch(setPaginatedDocuments(data ? data.docs : []));
         setTotalPages(data ? data.totalPages : 0);
-        setTotalDocuments(data ? data.totalDocuments : 0);
+        //setTotalDocuments(data ? data.totalDocuments : 0);
       })
       .catch((error) => console.error(error))
       .finally(() => {
@@ -258,13 +260,12 @@ const Documents: React.FC<DocumentsProps> = ({ handleDeleteDocument }) => {
         rowsPerPage={rowsPerPage}
         onPageChange={(page) => {
           setCurrentPage(page);
-          refreshDocs(sortField, page, rowsPerPage); // Pass `true` to reset lastID if not using cursor
+          refreshDocs(sortField, page, rowsPerPage);
         }}
         onRowsPerPageChange={(rows) => {
-          console.log('Pagination - Rows per Page Change:', rows);
           setRowsPerPage(rows);
-          setCurrentPage(1); // Reset to page 1 on rows per page change
-          refreshDocs(sortField, 1, rows); // Reset lastID for fresh pagination
+          setCurrentPage(1);
+          refreshDocs(sortField, 1, rows);
         }}
       />
     </div>
diff --git a/frontend/src/settings/index.tsx b/frontend/src/settings/index.tsx
index 24ea26812..15c7ce086 100644
--- a/frontend/src/settings/index.tsx
+++ b/frontend/src/settings/index.tsx
@@ -8,6 +8,8 @@ import i18n from '../locale/i18n';
 import { Doc } from '../models/misc';
 import {
   selectSourceDocs,
+  selectPaginatedDocuments,
+  setPaginatedDocuments,
   setSourceDocs,
 } from '../preferences/preferenceSlice';
 import Analytics from './Analytics';
@@ -26,20 +28,29 @@ export default function Settings() {
   );
 
   const documents = useSelector(selectSourceDocs);
+  const paginatedDocuments = useSelector(selectPaginatedDocuments);
   const updateWidgetScreenshot = (screenshot: File | null) => {
     setWidgetScreenshot(screenshot);
   };
 
+  const updateDocumentsList = (documents: Doc[], index: number) => [
+    ...documents.slice(0, index),
+    ...documents.slice(index + 1),
+  ];
+
   const handleDeleteClick = (index: number, doc: Doc) => {
     userService
       .deletePath(doc.id ?? '')
       .then((response) => {
         if (response.ok && documents) {
-          const updatedDocuments = [
-            ...documents.slice(0, index),
-            ...documents.slice(index + 1),
-          ];
-          dispatch(setSourceDocs(updatedDocuments));
+          if (paginatedDocuments) {
+            dispatch(
+              setPaginatedDocuments(
+                updateDocumentsList(paginatedDocuments, index),
+              ),
+            );
+          }
+          dispatch(setSourceDocs(updateDocumentsList(documents, index)));
         }
       })
       .catch((error) => console.error(error));
@@ -70,7 +81,12 @@ export default function Settings() {
       case t('settings.general.label'):
         return <General />;
       case t('settings.documents.label'):
-        return <Documents handleDeleteDocument={handleDeleteClick} />;
+        return (
+          <Documents
+            paginatedDocuments={paginatedDocuments}
+            handleDeleteDocument={handleDeleteClick}
+          />
+        );
       case 'Widgets':
         return (
           <Widgets

From 6c585de6d384c1931ac7630e57cd414f2c5ac19d Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sun, 10 Nov 2024 14:53:52 -0500
Subject: [PATCH 29/42] add paginated to store, and fix upload docs property

---
 frontend/src/store.ts          | 1 +
 frontend/src/upload/Upload.tsx | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/frontend/src/store.ts b/frontend/src/store.ts
index 5843d4932..8f426ed61 100644
--- a/frontend/src/store.ts
+++ b/frontend/src/store.ts
@@ -38,6 +38,7 @@ const preloadedState: { preference: Preference } = {
       },
     ],
     modalState: 'INACTIVE',
+    paginatedDocuments: null,
   },
 };
 const store = configureStore({
diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx
index 443127c05..4f31ea0ba 100644
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -191,8 +191,8 @@ function Upload({
                         )) ||
                         [],
                     );
-                    if (data && Array.isArray(data.docs)) {
-                      data.docs.map((updatedDoc: Doc) => {
+                    if (data && Array.isArray(data)) {
+                      data.map((updatedDoc: Doc) => {
                         if (updatedDoc.id && !docIds.has(updatedDoc.id)) {
                           // Select the doc not present in the intersection of current Docs and fetched data
                           dispatch(setSelectedDocs(updatedDoc));

From 32c67c2a0213a829a02f366084c41024f7e45cd9 Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sun, 10 Nov 2024 15:33:19 -0500
Subject: [PATCH 30/42] add property sync and etc ... to align with original
 source

---
 application/api/user/routes.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/application/api/user/routes.py b/application/api/user/routes.py
index 46ed15cde..37a3f7a15 100644
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -465,10 +465,17 @@ def get(self):
 
             paginated_docs = []
             for doc in documents:
-                print(doc)
-                doc["id"] = str(doc["_id"])
-                del doc["_id"]
-                paginated_docs.append(doc)
+                doc_data = {
+                    "id": str(doc["_id"]),
+                    "name": doc.get("name", ""),
+                    "date": doc.get("date", ""),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "location": "local",
+                    "tokens": doc.get("tokens", ""),
+                    "retriever": doc.get("retriever", "classic"),
+                    "syncFrequency": doc.get("sync_frequency", ""),
+                }
+                paginated_docs.append(doc_data)
 
             response = {
                 "total": total_documents,
@@ -499,9 +506,7 @@ def get(self):
         ]
 
         try:
-            for index in sources_collection.find({"user": user}).sort(
-                "date", -1
-            ):
+            for index in sources_collection.find({"user": user}).sort("date", -1):
                 data.append(
                     {
                         "id": str(index["_id"]),

From 6974db5fd89711331a4565ae7a7cc4ee7619f06a Mon Sep 17 00:00:00 2001
From: fadingNA <nplodthong@myseneca.ca>
Date: Sun, 10 Nov 2024 15:33:33 -0500
Subject: [PATCH 31/42] fix manage sync

---
 frontend/src/index.css              |  2 +-
 frontend/src/settings/Documents.tsx | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/frontend/src/index.css b/frontend/src/index.css
index 6de42c244..7c9cc02ce 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -54,7 +54,7 @@ body.dark {
   }
 
   .table-default th {
-    @apply p-4 w-1/3 font-normal text-gray-400 text-nowrap; /* Remove border-r */
+    @apply p-4 w-full font-normal text-gray-400 text-nowrap; /* Remove border-r */
   }
 
   .table-default th:last-child {
diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx
index eaee879e5..f91a33559 100644
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -58,6 +58,7 @@ const Documents: React.FC<DocumentsProps> = ({
   );
   // State for documents
   const currentDocuments = filteredDocuments ?? [];
+  console.log('currentDocuments', currentDocuments);
   const syncOptions = [
     { label: 'Never', value: 'never' },
     { label: 'Daily', value: 'daily' },
@@ -101,12 +102,25 @@ const Documents: React.FC<DocumentsProps> = ({
     userService
       .manageSync({ source_id: doc.id, sync_frequency })
       .then(() => {
+        // First, fetch the updated source docs
         return getDocs();
       })
       .then((data) => {
         dispatch(setSourceDocs(data));
+        return getDocsWithPagination(
+          sortField,
+          sortOrder,
+          currentPage,
+          rowsPerPage,
+        );
       })
-      .catch((error) => console.error(error))
+      .then((paginatedData) => {
+        dispatch(
+          setPaginatedDocuments(paginatedData ? paginatedData.docs : []),
+        );
+        setTotalPages(paginatedData ? paginatedData.totalPages : 0);
+      })
+      .catch((error) => console.error('Error in handleManageSync:', error))
       .finally(() => {
         setLoading(false);
       });

From ebf61092193b13af5df054823cff1be2faad90d9 Mon Sep 17 00:00:00 2001
From: Alex <a@tushynski.me>
Date: Mon, 11 Nov 2024 22:53:00 +0000
Subject: [PATCH 32/42] feat: medium size widget docs page

---
 docs/pages/_app.mdx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/pages/_app.mdx b/docs/pages/_app.mdx
index ac2be195e..1cb8cadd6 100644
--- a/docs/pages/_app.mdx
+++ b/docs/pages/_app.mdx
@@ -4,7 +4,7 @@ export default function MyApp({ Component, pageProps }) {
   return (
     <>
       <Component {...pageProps} />
-        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" theme="dark" />
+        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" theme="dark" size="medium" />
     </>
   )
 }
\ No newline at end of file

From 9bbe7564a99c798a14fd9bb670e5eb123074502b Mon Sep 17 00:00:00 2001
From: Alex <a@tushynski.me>
Date: Wed, 13 Nov 2024 11:45:42 +0000
Subject: [PATCH 33/42] fix: llamacpp

---
 application/api/answer/routes.py     | 1 +
 application/retriever/classic_rag.py | 3 ---
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py
index 182cdf2b3..f109db268 100644
--- a/application/api/answer/routes.py
+++ b/application/api/answer/routes.py
@@ -241,6 +241,7 @@ def complete_stream(
         yield f"data: {data}\n\n"
     except Exception as e:
         print("\033[91merr", str(e), file=sys.stderr)
+        traceback.print_exc()
         data = json.dumps(
             {
                 "type": "error",
diff --git a/application/retriever/classic_rag.py b/application/retriever/classic_rag.py
index 6a67cb380..42e318d20 100644
--- a/application/retriever/classic_rag.py
+++ b/application/retriever/classic_rag.py
@@ -45,7 +45,6 @@ def _get_data(self):
                 settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
             )
             docs_temp = docsearch.search(self.question, k=self.chunks)
-            print(docs_temp)
             docs = [
                 {
                     "title": i.metadata.get(
@@ -60,8 +59,6 @@ def _get_data(self):
                 }
                 for i in docs_temp
             ]
-        if settings.LLM_NAME == "llama.cpp":
-            docs = [docs[0]]
 
         return docs
 

From 6b9519b56f64fd9df80422a98fce9cf5ac85f263 Mon Sep 17 00:00:00 2001
From: Mason Hu <mason.hu@canonical.com>
Date: Wed, 13 Nov 2024 14:36:34 +0200
Subject: [PATCH 34/42] fix: prevent previous prompt name and content from
 displaying when adding new prompt

---
 frontend/src/settings/Prompts.tsx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frontend/src/settings/Prompts.tsx b/frontend/src/settings/Prompts.tsx
index 3d0415550..9501b47dd 100644
--- a/frontend/src/settings/Prompts.tsx
+++ b/frontend/src/settings/Prompts.tsx
@@ -58,7 +58,8 @@ export default function Prompts({
       }
       setModalState('INACTIVE');
       onSelectPrompt(newPromptName, newPrompt.id, newPromptContent);
-      setNewPromptName(newPromptName);
+      setNewPromptName('');
+      setNewPromptContent('');
     } catch (error) {
       console.error(error);
     }

From 2fe3cb2b22a999bff70c668172748b8b2ec0d330 Mon Sep 17 00:00:00 2001
From: Mason Hu <mason.hu@canonical.com>
Date: Wed, 13 Nov 2024 14:51:48 +0200
Subject: [PATCH 35/42] fix: hide edit option for default prompts

---
 frontend/src/components/Dropdown.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/components/Dropdown.tsx b/frontend/src/components/Dropdown.tsx
index 07f33650a..15923661b 100644
--- a/frontend/src/components/Dropdown.tsx
+++ b/frontend/src/components/Dropdown.tsx
@@ -140,7 +140,7 @@ function Dropdown({
                             : option.description
                         }`}
               </span>
-              {showEdit && onEdit && (
+              {showEdit && onEdit && option.type !== 'public' && (
                 <img
                   src={Edit}
                   alt="Edit"

From f6e9f9011d700bbdc22482a2c28ee38234ee071a Mon Sep 17 00:00:00 2001
From: Mason Hu <mason.hu@canonical.com>
Date: Wed, 13 Nov 2024 15:28:16 +0200
Subject: [PATCH 36/42] fix: prevent saving prompt with same name as existing
 prompt

---
 frontend/src/preferences/PromptsModal.tsx | 41 ++++++++++++++++++++---
 frontend/src/settings/Prompts.tsx         |  1 +
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/frontend/src/preferences/PromptsModal.tsx b/frontend/src/preferences/PromptsModal.tsx
index c207e0be4..3aa8c54cf 100644
--- a/frontend/src/preferences/PromptsModal.tsx
+++ b/frontend/src/preferences/PromptsModal.tsx
@@ -1,6 +1,7 @@
 import { ActiveState } from '../models/misc';
 import Exit from '../assets/exit.svg';
 import Input from '../components/Input';
+import React from 'react';
 
 function AddPrompt({
   setModalState,
@@ -9,6 +10,7 @@ function AddPrompt({
   setNewPromptName,
   newPromptContent,
   setNewPromptContent,
+  disableSave,
 }: {
   setModalState: (state: ActiveState) => void;
   handleAddPrompt?: () => void;
@@ -16,6 +18,7 @@ function AddPrompt({
   setNewPromptName: (name: string) => void;
   newPromptContent: string;
   setNewPromptContent: (content: string) => void;
+  disableSave: boolean;
 }) {
   return (
     <div className="relative">
@@ -23,6 +26,8 @@ function AddPrompt({
         className="absolute top-3 right-4 m-2 w-3"
         onClick={() => {
           setModalState('INACTIVE');
+          setNewPromptName('');
+          setNewPromptContent('');
         }}
       >
         <img className="filter dark:invert" src={Exit} />
@@ -41,7 +46,7 @@ function AddPrompt({
             className="h-10 rounded-lg"
             value={newPromptName}
             onChange={(e) => setNewPromptName(e.target.value)}
-          ></Input>
+          />
           <div className="relative bottom-12 left-3 mt-[-3.00px]">
             <span className="bg-white px-1 text-xs text-silver dark:bg-outer-space dark:text-silver">
               Prompt Name
@@ -62,6 +67,8 @@ function AddPrompt({
           <button
             onClick={handleAddPrompt}
             className="rounded-3xl bg-purple-30 px-5 py-2 text-sm text-white transition-all hover:opacity-90"
+            disabled={disableSave}
+            title={disableSave && newPromptName ? 'Name already exists' : ''}
           >
             Save
           </button>
@@ -79,6 +86,7 @@ function EditPrompt({
   editPromptContent,
   setEditPromptContent,
   currentPromptEdit,
+  disableSave,
 }: {
   setModalState: (state: ActiveState) => void;
   handleEditPrompt?: (id: string, type: string) => void;
@@ -87,6 +95,7 @@ function EditPrompt({
   editPromptContent: string;
   setEditPromptContent: (content: string) => void;
   currentPromptEdit: { name: string; id: string; type: string };
+  disableSave: boolean;
 }) {
   return (
     <div className="relative">
@@ -140,7 +149,8 @@ function EditPrompt({
               handleEditPrompt &&
                 handleEditPrompt(currentPromptEdit.id, currentPromptEdit.type);
             }}
-            disabled={currentPromptEdit.type === 'public'}
+            disabled={currentPromptEdit.type === 'public' || disableSave}
+            title={disableSave && editPromptName ? 'Name already exists' : ''}
           >
             Save
           </button>
@@ -151,6 +161,7 @@ function EditPrompt({
 }
 
 export default function PromptsModal({
+  existingPrompts,
   modalState,
   setModalState,
   type,
@@ -166,6 +177,7 @@ export default function PromptsModal({
   handleAddPrompt,
   handleEditPrompt,
 }: {
+  existingPrompts: { name: string; id: string; type: string }[];
   modalState: ActiveState;
   setModalState: (state: ActiveState) => void;
   type: 'ADD' | 'EDIT';
@@ -181,6 +193,25 @@ export default function PromptsModal({
   handleAddPrompt?: () => void;
   handleEditPrompt?: (id: string, type: string) => void;
 }) {
+  const [disableSave, setDisableSave] = React.useState(true);
+  const handlePrompNameChange = (edit: boolean, newName: string) => {
+    const nameExists = existingPrompts.find(
+      (prompt) => newName === prompt.name,
+    );
+
+    if (newName && !nameExists) {
+      setDisableSave(false);
+    } else {
+      setDisableSave(true);
+    }
+
+    if (edit) {
+      setEditPromptName(newName);
+    } else {
+      setNewPromptName(newName);
+    }
+  };
+
   let view;
 
   if (type === 'ADD') {
@@ -189,9 +220,10 @@ export default function PromptsModal({
         setModalState={setModalState}
         handleAddPrompt={handleAddPrompt}
         newPromptName={newPromptName}
-        setNewPromptName={setNewPromptName}
+        setNewPromptName={handlePrompNameChange.bind(null, false)}
         newPromptContent={newPromptContent}
         setNewPromptContent={setNewPromptContent}
+        disableSave={disableSave}
       />
     );
   } else if (type === 'EDIT') {
@@ -200,10 +232,11 @@ export default function PromptsModal({
         setModalState={setModalState}
         handleEditPrompt={handleEditPrompt}
         editPromptName={editPromptName}
-        setEditPromptName={setEditPromptName}
+        setEditPromptName={handlePrompNameChange.bind(null, true)}
         editPromptContent={editPromptContent}
         setEditPromptContent={setEditPromptContent}
         currentPromptEdit={currentPromptEdit}
+        disableSave={disableSave}
       />
     );
   } else {
diff --git a/frontend/src/settings/Prompts.tsx b/frontend/src/settings/Prompts.tsx
index 9501b47dd..6e1810e52 100644
--- a/frontend/src/settings/Prompts.tsx
+++ b/frontend/src/settings/Prompts.tsx
@@ -179,6 +179,7 @@ export default function Prompts({
         </div>
       </div>
       <PromptsModal
+        existingPrompts={prompts}
         type={modalType}
         modalState={modalState}
         setModalState={setModalState}

From bd636d59ddf6d08868b882aa69a61c69e78181cb Mon Sep 17 00:00:00 2001
From: Siddhant Rai <siddhant.rai.5686@gmail.com>
Date: Thu, 14 Nov 2024 15:35:22 +0530
Subject: [PATCH 37/42] refactor: smooth answer appearance

---
 .../src/conversation/ConversationBubble.tsx   | 17 +++++++-----
 frontend/src/index.css                        | 26 +++++++++++++++++++
 2 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx
index 284c2b568..8e5df6662 100644
--- a/frontend/src/conversation/ConversationBubble.tsx
+++ b/frontend/src/conversation/ConversationBubble.tsx
@@ -1,12 +1,14 @@
+import 'katex/dist/katex.min.css';
+
 import { forwardRef, useState } from 'react';
 import ReactMarkdown from 'react-markdown';
 import { useSelector } from 'react-redux';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism';
+import rehypeKatex from 'rehype-katex';
 import remarkGfm from 'remark-gfm';
 import remarkMath from 'remark-math';
-import rehypeKatex from 'rehype-katex';
-import 'katex/dist/katex.min.css';
+
 import DocsGPT3 from '../assets/cute_docsgpt3.svg';
 import Dislike from '../assets/dislike.svg?react';
 import Document from '../assets/document.svg';
@@ -16,13 +18,13 @@ import Sources from '../assets/sources.svg';
 import Avatar from '../components/Avatar';
 import CopyButton from '../components/CopyButton';
 import Sidebar from '../components/Sidebar';
+import SpeakButton from '../components/TextToSpeechButton';
 import {
   selectChunks,
   selectSelectedDocs,
 } from '../preferences/preferenceSlice';
 import classes from './ConversationBubble.module.css';
 import { FEEDBACK, MESSAGE_TYPE } from './conversationModels';
-import SpeakButton from '../components/TextToSpeechButton';
 
 const DisableSourceFE = import.meta.env.VITE_DISABLE_SOURCE_FE || false;
 
@@ -41,6 +43,7 @@ const ConversationBubble = forwardRef<
   { message, type, className, feedback, handleFeedback, sources, retryBtn },
   ref,
 ) {
+  // const bubbleRef = useRef<HTMLDivElement | null>(null);
   const chunks = useSelector(selectChunks);
   const selectedDocs = useSelector(selectSelectedDocs);
   const [isLikeHovered, setIsLikeHovered] = useState(false);
@@ -141,7 +144,7 @@ const ConversationBubble = forwardRef<
                 />
                 <p className="text-base font-semibold">Sources</p>
               </div>
-              <div className="ml-3 mr-5 max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
+              <div className="fade-in ml-3 mr-5 max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
                 <div className="grid grid-cols-2 gap-2 lg:grid-cols-4">
                   {sources?.slice(0, 3)?.map((source, index) => (
                     <div key={index} className="relative">
@@ -190,7 +193,7 @@ const ConversationBubble = forwardRef<
                       </div>
                       {activeTooltip === index && (
                         <div
-                          className={`absolute left-1/2 z-30 max-h-48 w-40 translate-x-[-50%] translate-y-[3px] rounded-xl bg-[#FBFBFB] p-4 text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver sm:w-56`}
+                          className={`absolute left-1/2 z-50 max-h-48 w-40 translate-x-[-50%] translate-y-[3px] rounded-xl bg-[#FBFBFB] p-4 text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver sm:w-56`}
                           onMouseOver={() => setActiveTooltip(index)}
                           onMouseOut={() => setActiveTooltip(null)}
                         >
@@ -231,14 +234,14 @@ const ConversationBubble = forwardRef<
             <p className="text-base font-semibold">Answer</p>
           </div>
           <div
-            className={`ml-2 mr-5 flex max-w-[90vw] rounded-[28px] bg-gray-1000 py-[14px] px-7 dark:bg-gun-metal md:max-w-[70vw] lg:max-w-[50vw] ${
+            className={`fade-in-bubble ml-2 mr-5 flex max-w-[90vw] rounded-[28px] bg-gray-1000 py-[14px] px-7 dark:bg-gun-metal md:max-w-[70vw] lg:max-w-[50vw] ${
               type === 'ERROR'
                 ? 'relative flex-row items-center rounded-full border border-transparent bg-[#FFE7E7] p-2 py-5 text-sm font-normal text-red-3000  dark:border-red-2000 dark:text-white'
                 : 'flex-col rounded-3xl'
             }`}
           >
             <ReactMarkdown
-              className="whitespace-pre-wrap break-normal leading-normal"
+              className="fade-in whitespace-pre-wrap break-normal leading-normal"
               remarkPlugins={[remarkGfm, remarkMath]}
               rehypePlugins={[rehypeKatex]}
               components={{
diff --git a/frontend/src/index.css b/frontend/src/index.css
index 7c9cc02ce..9bd2ec964 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -514,3 +514,29 @@ input:-webkit-autofill:focus {
 .logs-table {
   font-family: 'IBMPlexMono-Medium', system-ui;
 }
+
+.fade-in {
+  animation: fadeIn 0.5s ease-in-out;
+}
+
+@keyframes fadeIn {
+  0% {
+    opacity: 0;
+  }
+  100% {
+    opacity: 1;
+  }
+}
+
+.fade-in-bubble {
+  opacity: 0;
+  transform: translateY(10px);
+  animation: fadeInUp 0.5s forwards;
+}
+
+@keyframes fadeInUp {
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}

From ed8d553491f80ec30ed24378a1f946521c4e7e9e Mon Sep 17 00:00:00 2001
From: rohittcodes <rohittcodes@gmail.com>
Date: Thu, 14 Nov 2024 20:46:02 +0530
Subject: [PATCH 38/42] refactor: loader-color

---
 frontend/src/upload/Upload.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx
index 4fee88f6d..2f28042a3 100644
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -76,7 +76,7 @@ function Upload({
         <div className="relative w-32 h-32 rounded-full">
           <div className="absolute inset-0 rounded-full shadow-[0_0_10px_2px_rgba(0,0,0,0.3)_inset] dark:shadow-[0_0_10px_2px_rgba(0,0,0,0.3)_inset]"></div>
           <div
-            className={`absolute inset-0 rounded-full ${progressPercent === 100 ? 'shadow-xl shadow-lime-300/50 dark:shadow-lime-300/50 bg-gradient-to-r from-white to-gray-400 dark:bg-gradient-to-br dark:from-gray-500 dark:to-gray-300' : 'shadow-[0_2px_0_#FF3D00_inset] dark:shadow-[0_2px_0_#FF3D00_inset]'}`}
+            className={`absolute inset-0 rounded-full ${progressPercent === 100 ? 'shadow-xl shadow-lime-300/50 dark:shadow-lime-300/50 bg-gradient-to-r from-white to-gray-400 dark:bg-gradient-to-br dark:from-gray-500 dark:to-gray-300' : 'shadow-[0_4px_0_#7D54D1] dark:shadow-[0_4px_0_#7D54D1]'}`}
             style={{
               animation: `${progressPercent === 100 ? 'none' : 'rotate 2s linear infinite'}`,
             }}

From 2245f4690eb7aa111508b72c8f615f0d0b7d949e Mon Sep 17 00:00:00 2001
From: Alex <a@tushynski.me>
Date: Fri, 15 Nov 2024 11:02:27 +0000
Subject: [PATCH 39/42] fix: reddit loader validation

---
 application/parser/remote/reddit_loader.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/application/parser/remote/reddit_loader.py b/application/parser/remote/reddit_loader.py
index 0230653ae..22f5b185e 100644
--- a/application/parser/remote/reddit_loader.py
+++ b/application/parser/remote/reddit_loader.py
@@ -1,10 +1,19 @@
 from application.parser.remote.base import BaseRemote
 from langchain_community.document_loaders import RedditPostsLoader
+import json
 
 
 class RedditPostsLoaderRemote(BaseRemote):
     def load_data(self, inputs):
-        data = eval(inputs)
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        required_fields = ["client_id", "client_secret", "user_agent", "search_queries"]
+        missing_fields = [field for field in required_fields if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
         client_id = data.get("client_id")
         client_secret = data.get("client_secret")
         user_agent = data.get("user_agent")

From a4c95fd62b8e6e731b40b47a87ebb6f4255bfade Mon Sep 17 00:00:00 2001
From: Alex <a@tushynski.me>
Date: Fri, 15 Nov 2024 12:17:25 +0000
Subject: [PATCH 40/42] feat: add google ai

---
 application/llm/google_ai.py   | 52 ++++++++++++++++++++++++++++++++++
 application/llm/llm_creator.py |  4 ++-
 2 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 application/llm/google_ai.py

diff --git a/application/llm/google_ai.py b/application/llm/google_ai.py
new file mode 100644
index 000000000..ea460345d
--- /dev/null
+++ b/application/llm/google_ai.py
@@ -0,0 +1,52 @@
+from application.llm.base import BaseLLM
+from application.core.settings import settings
+
+
+
+
+class GoogleLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _clean_messages_google(self, messages):
+        return [
+            {
+                "role": "model" if message["role"] == "system" else message["role"],
+                "parts": [message["content"]],
+            }
+            for message in messages[1:]
+        ]
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        import google.generativeai as genai
+        genai.configure(api_key=self.api_key)
+        model = genai.GenerativeModel(model, system_instruction=messages[0]["content"])
+        response = model.generate_content(self._clean_messages_google(messages))
+        return response.text
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        import google.generativeai as genai
+        genai.configure(api_key=self.api_key)
+        model = genai.GenerativeModel(model, system_instruction=messages[0]["content"])
+        response = model.generate_content(self._clean_messages_google(messages), stream=True)
+        for line in response:
+            if line.text is not None:
+                yield line.text
\ No newline at end of file
diff --git a/application/llm/llm_creator.py b/application/llm/llm_creator.py
index 6a19de107..f32089de8 100644
--- a/application/llm/llm_creator.py
+++ b/application/llm/llm_creator.py
@@ -6,6 +6,7 @@
 from application.llm.anthropic import AnthropicLLM
 from application.llm.docsgpt_provider import DocsGPTAPILLM
 from application.llm.premai import PremAILLM
+from application.llm.google_ai import GoogleLLM
 
 
 class LLMCreator:
@@ -18,7 +19,8 @@ class LLMCreator:
         "anthropic": AnthropicLLM,
         "docsgpt": DocsGPTAPILLM,
         "premai": PremAILLM,
-        "groq": GroqLLM
+        "groq": GroqLLM,
+        "google": GoogleLLM
     }
 
     @classmethod

From 144ab61e079cb37fa93baa4095d5ae569b66adbf Mon Sep 17 00:00:00 2001
From: Alex <a@tushynski.me>
Date: Fri, 15 Nov 2024 12:19:43 +0000
Subject: [PATCH 41/42] fix: ruff lint

---
 application/llm/google_ai.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/application/llm/google_ai.py b/application/llm/google_ai.py
index ea460345d..df252abfc 100644
--- a/application/llm/google_ai.py
+++ b/application/llm/google_ai.py
@@ -1,8 +1,4 @@
 from application.llm.base import BaseLLM
-from application.core.settings import settings
-
-
-
 
 class GoogleLLM(BaseLLM):
 

From 47d687b1518ca458ac5217860f5e50aa252ad2d8 Mon Sep 17 00:00:00 2001
From: Siddhant Rai <siddhant.rai.5686@gmail.com>
Date: Sat, 16 Nov 2024 17:13:15 +0530
Subject: [PATCH 42/42] fix: old streaming answer gets appended to new
 conversation

---
 frontend/src/Navigation.tsx                    |  2 ++
 frontend/src/conversation/conversationSlice.ts | 16 +++++++++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx
index 324d5aa05..242efb1a9 100644
--- a/frontend/src/Navigation.tsx
+++ b/frontend/src/Navigation.tsx
@@ -18,6 +18,7 @@ import SourceDropdown from './components/SourceDropdown';
 import {
   setConversation,
   updateConversationId,
+  handleAbort,
 } from './conversation/conversationSlice';
 import ConversationTile from './conversation/ConversationTile';
 import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks';
@@ -180,6 +181,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
   };
 
   const resetConversation = () => {
+    handleAbort();
     dispatch(setConversation([]));
     dispatch(
       updateConversationId({
diff --git a/frontend/src/conversation/conversationSlice.ts b/frontend/src/conversation/conversationSlice.ts
index 3ca05c7b8..5a78e0141 100644
--- a/frontend/src/conversation/conversationSlice.ts
+++ b/frontend/src/conversation/conversationSlice.ts
@@ -17,9 +17,23 @@ const initialState: ConversationState = {
 
 const API_STREAMING = import.meta.env.VITE_API_STREAMING === 'true';
 
+let abortController: AbortController | null = null;
+export function handleAbort() {
+  if (abortController) {
+    abortController.abort();
+    abortController = null;
+  }
+}
+
 export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
   'fetchAnswer',
-  async ({ question }, { dispatch, getState, signal }) => {
+  async ({ question }, { dispatch, getState }) => {
+    if (abortController) {
+      abortController.abort();
+    }
+    abortController = new AbortController();
+    const { signal } = abortController;
+
     let isSourceUpdated = false;
     const state = getState() as RootState;
     if (state.preference) {