Merge pull request #5861 from open-webui/projects

feat: knowledge/projects

backend/open_webui/apps/retrieval/main.py

@@ -1,3 +1,5 @@
+# TODO: Merge this with the webui_app and make it a single app
+
 import json
 import logging
 import mimetypes
@@ -634,9 +636,23 @@ def save_docs_to_vector_db(
     metadata: Optional[dict] = None,
     overwrite: bool = False,
     split: bool = True,
+    add: bool = False,
 ) -> bool:
     log.info(f"save_docs_to_vector_db {docs} {collection_name}")
 
+    # Check if entries with the same hash (metadata.hash) already exist
+    if metadata and "hash" in metadata:
+        result = VECTOR_DB_CLIENT.query(
+            collection_name=collection_name,
+            filter={"hash": metadata["hash"]},
+        )
+
+        if result:
+            existing_doc_ids = result.ids[0]
+            if existing_doc_ids:
+                log.info(f"Document with hash {metadata['hash']} already exists")
+                raise ValueError(ERROR_MESSAGES.DUPLICATE_CONTENT)
+
     if split:
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=app.state.config.CHUNK_SIZE,
@@ -659,42 +675,46 @@ def save_docs_to_vector_db(
                 metadata[key] = str(value)
 
     try:
-        if overwrite:
-            if VECTOR_DB_CLIENT.has_collection(collection_name=collection_name):
-                log.info(f"deleting existing collection {collection_name}")
-                VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name)
-
         if VECTOR_DB_CLIENT.has_collection(collection_name=collection_name):
             log.info(f"collection {collection_name} already exists")
-            return True
-        else:
-            embedding_function = get_embedding_function(
-                app.state.config.RAG_EMBEDDING_ENGINE,
-                app.state.config.RAG_EMBEDDING_MODEL,
-                app.state.sentence_transformer_ef,
-                app.state.config.OPENAI_API_KEY,
-                app.state.config.OPENAI_API_BASE_URL,
-                app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE,
-            )
 
-            embeddings = embedding_function(
+            if overwrite:
-                list(map(lambda x: x.replace("\n", " "), texts))
+                VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name)
-            )
+                log.info(f"deleting existing collection {collection_name}")
 
-            VECTOR_DB_CLIENT.insert(
+            if add is False:
-                collection_name=collection_name,
+                return True
-                items=[
-                    {
-                        "id": str(uuid.uuid4()),
-                        "text": text,
-                        "vector": embeddings[idx],
-                        "metadata": metadatas[idx],
-                    }
-                    for idx, text in enumerate(texts)
-                ],
-            )
 
-            return True
+        log.info(f"adding to collection {collection_name}")
+        embedding_function = get_embedding_function(
+            app.state.config.RAG_EMBEDDING_ENGINE,
+            app.state.config.RAG_EMBEDDING_MODEL,
+            app.state.sentence_transformer_ef,
+            app.state.config.OPENAI_API_KEY,
+            app.state.config.OPENAI_API_BASE_URL,
+            app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE,
+        )
+
+        embeddings = embedding_function(
+            list(map(lambda x: x.replace("\n", " "), texts))
+        )
+
+        items = [
+            {
+                "id": str(uuid.uuid4()),
+                "text": text,
+                "vector": embeddings[idx],
+                "metadata": metadatas[idx],
+            }
+            for idx, text in enumerate(texts)
+        ]
+
+        VECTOR_DB_CLIENT.insert(
+            collection_name=collection_name,
+            items=items,
+        )
+
+        return True
     except Exception as e:
         log.exception(e)
         return False
@@ -702,6 +722,7 @@ def save_docs_to_vector_db(
 
 class ProcessFileForm(BaseModel):
     file_id: str
+    content: Optional[str] = None
     collection_name: Optional[str] = None
 
 
@@ -712,42 +733,91 @@ def process_file(
 ):
     try:
         file = Files.get_file_by_id(form_data.file_id)
-        file_path = file.meta.get("path", f"{UPLOAD_DIR}/{file.filename}")
 
         collection_name = form_data.collection_name
         if collection_name is None:
-            with open(file_path, "rb") as f:
+            collection_name = f"file-{file.id}"
-                collection_name = calculate_sha256(f)[:63]
 
         loader = Loader(
             engine=app.state.config.CONTENT_EXTRACTION_ENGINE,
             TIKA_SERVER_URL=app.state.config.TIKA_SERVER_URL,
             PDF_EXTRACT_IMAGES=app.state.config.PDF_EXTRACT_IMAGES,
         )
-        docs = loader.load(file.filename, file.meta.get("content_type"), file_path)
-        text_content = " ".join([doc.page_content for doc in docs])
-        log.debug(f"text_content: {text_content}")
 
-        Files.update_files_metadata_by_id(
+        if form_data.content:
-            form_data.file_id,
+            docs = [
-            {
+                Document(
-                "content": {
+                    page_content=form_data.content,
-                    "text": text_content,
+                    metadata={
-                }
+                        "name": file.meta.get("name", file.filename),
-            },
+                        "created_by": file.user_id,
+                        **file.meta,
+                    },
+                )
+            ]
+
+            text_content = form_data.content
+        elif file.data.get("content", None):
+            docs = [
+                Document(
+                    page_content=file.data.get("content", ""),
+                    metadata={
+                        "name": file.meta.get("name", file.filename),
+                        "created_by": file.user_id,
+                        **file.meta,
+                    },
+                )
+            ]
+            text_content = file.data.get("content", "")
+        else:
+            file_path = file.meta.get("path", None)
+            if file_path:
+                docs = loader.load(
+                    file.filename, file.meta.get("content_type"), file_path
+                )
+            else:
+                docs = [
+                    Document(
+                        page_content=file.data.get("content", ""),
+                        metadata={
+                            "name": file.filename,
+                            "created_by": file.user_id,
+                            **file.meta,
+                        },
+                    )
+                ]
+
+            text_content = " ".join([doc.page_content for doc in docs])
+
+        log.debug(f"text_content: {text_content}")
+        Files.update_file_data_by_id(
+            file.id,
+            {"content": text_content},
         )
 
+        hash = calculate_sha256_string(text_content)
+        Files.update_file_hash_by_id(file.id, hash)
+
         try:
             result = save_docs_to_vector_db(
-                docs,
+                docs=docs,
-                collection_name,
+                collection_name=collection_name,
-                {
+                metadata={
-                    "file_id": form_data.file_id,
+                    "file_id": file.id,
                     "name": file.meta.get("name", file.filename),
+                    "hash": hash,
                 },
+                add=(True if form_data.collection_name else False),
             )
 
             if result:
+                Files.update_file_metadata_by_id(
+                    file.id,
+                    {
+                        "collection_name": collection_name,
+                    },
+                )
+
                 return {
                     "status": True,
                     "collection_name": collection_name,
@@ -755,10 +825,7 @@ def process_file(
                     "content": text_content,
                 }
         except Exception as e:
-            raise HTTPException(
+            raise e
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=e,
-            )
     except Exception as e:
         log.exception(e)
         if "No pandoc was found" in str(e):
@@ -769,7 +836,7 @@ def process_file(
         else:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
-                detail=ERROR_MESSAGES.DEFAULT(e),
+                detail=str(e),
             )
 
 
@@ -1183,6 +1250,30 @@ def query_collection_handler(
 ####################################
 
 
+class DeleteForm(BaseModel):
+    collection_name: str
+    file_id: str
+
+
+@app.post("/delete")
+def delete_entries_from_collection(form_data: DeleteForm, user=Depends(get_admin_user)):
+    try:
+        if VECTOR_DB_CLIENT.has_collection(collection_name=form_data.collection_name):
+            file = Files.get_file_by_id(form_data.file_id)
+            hash = file.hash
+
+            VECTOR_DB_CLIENT.delete(
+                collection_name=form_data.collection_name,
+                metadata={"hash": hash},
+            )
+            return {"status": True}
+        else:
+            return {"status": False}
+    except Exception as e:
+        log.exception(e)
+        return {"status": False}
+
+
 @app.post("/reset/db")
 def reset_vector_db(user=Depends(get_admin_user)):
     VECTOR_DB_CLIENT.reset()

backend/open_webui/apps/retrieval/utils.py

@@ -319,17 +319,25 @@ def get_rag_context(
     for file in files:
         if file.get("context") == "full":
             context = {
-                "documents": [[file.get("file").get("content")]],
+                "documents": [[file.get("file").get("data", {}).get("content")]],
                 "metadatas": [[{"file_id": file.get("id"), "name": file.get("name")}]],
             }
         else:
             context = None
 
-            collection_names = (
+            collection_names = []
-                file["collection_names"]
+            if file.get("type") == "collection":
-                if file["type"] == "collection"
+                if file.get("legacy"):
-                else [file["collection_name"]] if file["collection_name"] else []
+                    collection_names = file.get("collection_names", [])
-            )
+                else:
+                    collection_names.append(file["id"])
+            elif file.get("collection_name"):
+                collection_names.append(file["collection_name"])
+            elif file.get("id"):
+                if file.get("legacy"):
+                    collection_names.append(f"{file['id']}")
+                else:
+                    collection_names.append(f"file-{file['id']}")
 
             collection_names = set(collection_names).difference(extracted_collections)
             if not collection_names:

backend/open_webui/apps/retrieval/vector/dbs/chroma.py

@@ -49,22 +49,52 @@ class ChromaClient:
         self, collection_name: str, vectors: list[list[float | int]], limit: int
     ) -> Optional[SearchResult]:
         # Search for the nearest neighbor items based on the vectors and return 'limit' number of results.
-        collection = self.client.get_collection(name=collection_name)
+        try:
-        if collection:
+            collection = self.client.get_collection(name=collection_name)
-            result = collection.query(
+            if collection:
-                query_embeddings=vectors,
+                result = collection.query(
-                n_results=limit,
+                    query_embeddings=vectors,
-            )
+                    n_results=limit,
+                )
 
-            return SearchResult(
+                return SearchResult(
-                **{
+                    **{
-                    "ids": result["ids"],
+                        "ids": result["ids"],
-                    "distances": result["distances"],
+                        "distances": result["distances"],
-                    "documents": result["documents"],
+                        "documents": result["documents"],
-                    "metadatas": result["metadatas"],
+                        "metadatas": result["metadatas"],
-                }
+                    }
-            )
+                )
-        return None
+            return None
+        except Exception as e:
+            return None
+
+    def query(
+        self, collection_name: str, filter: dict, limit: int = 2
+    ) -> Optional[GetResult]:
+        # Query the items from the collection based on the filter.
+
+        try:
+            collection = self.client.get_collection(name=collection_name)
+            if collection:
+                result = collection.get(
+                    where=filter,
+                    limit=limit,
+                )
+
+                print(result)
+
+                return GetResult(
+                    **{
+                        "ids": [result["ids"]],
+                        "documents": [result["documents"]],
+                        "metadatas": [result["metadatas"]],
+                    }
+                )
+            return None
+        except Exception as e:
+            print(e)
+            return None
 
     def get(self, collection_name: str) -> Optional[GetResult]:
         # Get all the items in the collection.
@@ -111,11 +141,19 @@ class ChromaClient:
             ids=ids, documents=documents, embeddings=embeddings, metadatas=metadatas
         )
 
-    def delete(self, collection_name: str, ids: list[str]):
+    def delete(
+        self,
+        collection_name: str,
+        ids: Optional[list[str]] = None,
+        filter: Optional[dict] = None,
+    ):
         # Delete the items from the collection based on the ids.
         collection = self.client.get_collection(name=collection_name)
         if collection:
-            collection.delete(ids=ids)
+            if ids:
+                collection.delete(ids=ids)
+            elif filter:
+                collection.delete(where=filter)
 
     def reset(self):
         # Resets the database. This will delete all collections and item entries.

backend/open_webui/apps/retrieval/vector/dbs/milvus.py

@@ -135,6 +135,25 @@ class MilvusClient:
 
         return self._result_to_search_result(result)
 
+    def query(
+        self, collection_name: str, filter: dict, limit: int = 1
+    ) -> Optional[GetResult]:
+        # Query the items from the collection based on the filter.
+        filter_string = " && ".join(
+            [
+                f"JSON_CONTAINS(metadata[{key}], '{[value] if isinstance(value, str) else value}')"
+                for key, value in filter.items()
+            ]
+        )
+
+        result = self.client.query(
+            collection_name=f"{self.collection_prefix}_{collection_name}",
+            filter=filter_string,
+            limit=limit,
+        )
+
+        return self._result_to_get_result([result])
+
     def get(self, collection_name: str) -> Optional[GetResult]:
         # Get all the items in the collection.
         result = self.client.query(
@@ -187,13 +206,32 @@ class MilvusClient:
             ],
         )
 
-    def delete(self, collection_name: str, ids: list[str]):
+    def delete(
+        self,
+        collection_name: str,
+        ids: Optional[list[str]] = None,
+        filter: Optional[dict] = None,
+    ):
         # Delete the items from the collection based on the ids.
 
-        return self.client.delete(
+        if ids:
-            collection_name=f"{self.collection_prefix}_{collection_name}",
+            return self.client.delete(
-            ids=ids,
+                collection_name=f"{self.collection_prefix}_{collection_name}",
-        )
+                ids=ids,
+            )
+        elif filter:
+            # Convert the filter dictionary to a string using JSON_CONTAINS.
+            filter_string = " && ".join(
+                [
+                    f"JSON_CONTAINS(metadata[{key}], '{[value] if isinstance(value, str) else value}')"
+                    for key, value in filter.items()
+                ]
+            )
+
+            return self.client.delete(
+                collection_name=f"{self.collection_prefix}_{collection_name}",
+                filter=filter_string,
+            )
 
     def reset(self):
         # Resets the database. This will delete all collections and item entries.

backend/open_webui/apps/webui/main.py

@@ -10,11 +10,11 @@ from open_webui.apps.webui.routers import (
     auths,
     chats,
     configs,
-    documents,
     files,
     functions,
     memories,
     models,
+    knowledge,
     prompts,
     tools,
     users,
@@ -111,15 +111,15 @@ app.include_router(auths.router, prefix="/auths", tags=["auths"])
 app.include_router(users.router, prefix="/users", tags=["users"])
 app.include_router(chats.router, prefix="/chats", tags=["chats"])
 
-app.include_router(documents.router, prefix="/documents", tags=["documents"])
 app.include_router(models.router, prefix="/models", tags=["models"])
+app.include_router(knowledge.router, prefix="/knowledge", tags=["knowledge"])
 app.include_router(prompts.router, prefix="/prompts", tags=["prompts"])
 
-app.include_router(memories.router, prefix="/memories", tags=["memories"])
 app.include_router(files.router, prefix="/files", tags=["files"])
 app.include_router(tools.router, prefix="/tools", tags=["tools"])
 app.include_router(functions.router, prefix="/functions", tags=["functions"])
 
+app.include_router(memories.router, prefix="/memories", tags=["memories"])
 app.include_router(utils.router, prefix="/utils", tags=["utils"])
 
 

backend/open_webui/apps/webui/models/files.py

@@ -5,7 +5,7 @@ from typing import Optional
 from open_webui.apps.webui.internal.db import Base, JSONField, get_db
 from open_webui.env import SRC_LOG_LEVELS
 from pydantic import BaseModel, ConfigDict
-from sqlalchemy import BigInteger, Column, String, Text
+from sqlalchemy import BigInteger, Column, String, Text, JSON
 
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["MODELS"])
@@ -20,19 +20,29 @@ class File(Base):
 
     id = Column(String, primary_key=True)
     user_id = Column(String)
+    hash = Column(Text, nullable=True)
+
     filename = Column(Text)
+    data = Column(JSON, nullable=True)
     meta = Column(JSONField)
+
     created_at = Column(BigInteger)
+    updated_at = Column(BigInteger)
 
 
 class FileModel(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
     id: str
     user_id: str
-    filename: str
+    hash: Optional[str] = None
-    meta: dict
-    created_at: int  # timestamp in epoch
 
-    model_config = ConfigDict(from_attributes=True)
+    filename: str
+    data: Optional[dict] = None
+    meta: dict
+
+    created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
 
 
 ####################
@@ -43,14 +53,21 @@ class FileModel(BaseModel):
 class FileModelResponse(BaseModel):
     id: str
     user_id: str
+    hash: Optional[str] = None
+
     filename: str
+    data: Optional[dict] = None
     meta: dict
+
     created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
 
 
 class FileForm(BaseModel):
     id: str
+    hash: Optional[str] = None
     filename: str
+    data: dict = {}
     meta: dict = {}
 
 
@@ -62,6 +79,7 @@ class FilesTable:
                     **form_data.model_dump(),
                     "user_id": user_id,
                     "created_at": int(time.time()),
+                    "updated_at": int(time.time()),
                 }
             )
 
@@ -90,6 +108,13 @@ class FilesTable:
         with get_db() as db:
             return [FileModel.model_validate(file) for file in db.query(File).all()]
 
+    def get_files_by_ids(self, ids: list[str]) -> list[FileModel]:
+        with get_db() as db:
+            return [
+                FileModel.model_validate(file)
+                for file in db.query(File).filter(File.id.in_(ids)).all()
+            ]
+
     def get_files_by_user_id(self, user_id: str) -> list[FileModel]:
         with get_db() as db:
             return [
@@ -97,17 +122,38 @@ class FilesTable:
                 for file in db.query(File).filter_by(user_id=user_id).all()
             ]
 
-    def update_files_metadata_by_id(self, id: str, meta: dict) -> Optional[FileModel]:
+    def update_file_hash_by_id(self, id: str, hash: str) -> Optional[FileModel]:
         with get_db() as db:
             try:
                 file = db.query(File).filter_by(id=id).first()
-                file.meta = {**file.meta, **meta}
+                file.hash = hash
                 db.commit()
 
                 return FileModel.model_validate(file)
             except Exception:
                 return None
 
+    def update_file_data_by_id(self, id: str, data: dict) -> Optional[FileModel]:
+        with get_db() as db:
+            try:
+                file = db.query(File).filter_by(id=id).first()
+                file.data = {**(file.data if file.data else {}), **data}
+                db.commit()
+                return FileModel.model_validate(file)
+            except Exception as e:
+
+                return None
+
+    def update_file_metadata_by_id(self, id: str, meta: dict) -> Optional[FileModel]:
+        with get_db() as db:
+            try:
+                file = db.query(File).filter_by(id=id).first()
+                file.meta = {**(file.meta if file.meta else {}), **meta}
+                db.commit()
+                return FileModel.model_validate(file)
+            except Exception:
+                return None
+
     def delete_file_by_id(self, id: str) -> bool:
         with get_db() as db:
             try:

backend/open_webui/apps/webui/models/knowledge.py

@@ -0,0 +1,152 @@
+import json
+import logging
+import time
+from typing import Optional
+import uuid
+
+from open_webui.apps.webui.internal.db import Base, get_db
+from open_webui.env import SRC_LOG_LEVELS
+from pydantic import BaseModel, ConfigDict
+from sqlalchemy import BigInteger, Column, String, Text, JSON
+
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["MODELS"])
+
+####################
+# Knowledge DB Schema
+####################
+
+
+class Knowledge(Base):
+    __tablename__ = "knowledge"
+
+    id = Column(Text, unique=True, primary_key=True)
+    user_id = Column(Text)
+
+    name = Column(Text)
+    description = Column(Text)
+
+    data = Column(JSON, nullable=True)
+    meta = Column(JSON, nullable=True)
+
+    created_at = Column(BigInteger)
+    updated_at = Column(BigInteger)
+
+
+class KnowledgeModel(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+    id: str
+    user_id: str
+
+    name: str
+    description: str
+
+    data: Optional[dict] = None
+    meta: Optional[dict] = None
+
+    created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
+
+
+####################
+# Forms
+####################
+
+
+class KnowledgeResponse(BaseModel):
+    id: str
+    name: str
+    description: str
+    data: Optional[dict] = None
+    meta: Optional[dict] = None
+    created_at: int  # timestamp in epoch
+    updated_at: int  # timestamp in epoch
+
+
+class KnowledgeForm(BaseModel):
+    name: str
+    description: str
+    data: Optional[dict] = None
+
+
+class KnowledgeUpdateForm(BaseModel):
+    name: Optional[str] = None
+    description: Optional[str] = None
+    data: Optional[dict] = None
+
+
+class KnowledgeTable:
+    def insert_new_knowledge(
+        self, user_id: str, form_data: KnowledgeForm
+    ) -> Optional[KnowledgeModel]:
+        with get_db() as db:
+            knowledge = KnowledgeModel(
+                **{
+                    **form_data.model_dump(),
+                    "id": str(uuid.uuid4()),
+                    "user_id": user_id,
+                    "created_at": int(time.time()),
+                    "updated_at": int(time.time()),
+                }
+            )
+
+            try:
+                result = Knowledge(**knowledge.model_dump())
+                db.add(result)
+                db.commit()
+                db.refresh(result)
+                if result:
+                    return KnowledgeModel.model_validate(result)
+                else:
+                    return None
+            except Exception:
+                return None
+
+    def get_knowledge_items(self) -> list[KnowledgeModel]:
+        with get_db() as db:
+            return [
+                KnowledgeModel.model_validate(knowledge)
+                for knowledge in db.query(Knowledge)
+                .order_by(Knowledge.updated_at.desc())
+                .all()
+            ]
+
+    def get_knowledge_by_id(self, id: str) -> Optional[KnowledgeModel]:
+        try:
+            with get_db() as db:
+                knowledge = db.query(Knowledge).filter_by(id=id).first()
+                return KnowledgeModel.model_validate(knowledge) if knowledge else None
+        except Exception:
+            return None
+
+    def update_knowledge_by_id(
+        self, id: str, form_data: KnowledgeUpdateForm, overwrite: bool = False
+    ) -> Optional[KnowledgeModel]:
+        try:
+            with get_db() as db:
+                knowledge = self.get_knowledge_by_id(id=id)
+                db.query(Knowledge).filter_by(id=id).update(
+                    {
+                        **form_data.model_dump(exclude_none=True),
+                        "updated_at": int(time.time()),
+                    }
+                )
+                db.commit()
+                return self.get_knowledge_by_id(id=id)
+        except Exception as e:
+            log.exception(e)
+            return None
+
+    def delete_knowledge_by_id(self, id: str) -> bool:
+        try:
+            with get_db() as db:
+                db.query(Knowledge).filter_by(id=id).delete()
+                db.commit()
+                return True
+        except Exception:
+            return False
+
+
+Knowledges = KnowledgeTable()

backend/open_webui/apps/webui/routers/files.py

@@ -4,13 +4,18 @@ import shutil
 import uuid
 from pathlib import Path
 from typing import Optional
+from pydantic import BaseModel
 
 from open_webui.apps.webui.models.files import FileForm, FileModel, Files
+from open_webui.apps.retrieval.main import process_file, ProcessFileForm
+
 from open_webui.config import UPLOAD_DIR
 from open_webui.constants import ERROR_MESSAGES
 from open_webui.env import SRC_LOG_LEVELS
+
+
 from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status
-from fastapi.responses import FileResponse
+from fastapi.responses import FileResponse, StreamingResponse
 from open_webui.utils.utils import get_admin_user, get_verified_user
 
 log = logging.getLogger(__name__)
@@ -58,6 +63,13 @@ def upload_file(file: UploadFile = File(...), user=Depends(get_verified_user)):
             ),
         )
 
+        try:
+            process_file(ProcessFileForm(file_id=id))
+            file = Files.get_file_by_id(id=id)
+        except Exception as e:
+            log.exception(e)
+            log.error(f"Error processing file: {file.id}")
+
         if file:
             return file
         else:
@@ -143,6 +155,55 @@ async def get_file_by_id(id: str, user=Depends(get_verified_user)):
         )
 
 
+############################
+# Get File Data Content By Id
+############################
+
+
+@router.get("/{id}/data/content")
+async def get_file_data_content_by_id(id: str, user=Depends(get_verified_user)):
+    file = Files.get_file_by_id(id)
+
+    if file and (file.user_id == user.id or user.role == "admin"):
+        return {"content": file.data.get("content", "")}
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# Update File Data Content By Id
+############################
+
+
+class ContentForm(BaseModel):
+    content: str
+
+
+@router.post("/{id}/data/content/update")
+async def update_file_data_content_by_id(
+    id: str, form_data: ContentForm, user=Depends(get_verified_user)
+):
+    file = Files.get_file_by_id(id)
+
+    if file and (file.user_id == user.id or user.role == "admin"):
+        try:
+            process_file(ProcessFileForm(file_id=id, content=form_data.content))
+            file = Files.get_file_by_id(id=id)
+        except Exception as e:
+            log.exception(e)
+            log.error(f"Error processing file: {file.id}")
+
+        return {"content": file.data.get("content", "")}
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
 ############################
 # Get File Content By Id
 ############################
@@ -171,34 +232,37 @@ async def get_file_content_by_id(id: str, user=Depends(get_verified_user)):
         )
 
 
-@router.get("/{id}/content/text")
-async def get_file_text_content_by_id(id: str, user=Depends(get_verified_user)):
-    file = Files.get_file_by_id(id)
-
-    if file and (file.user_id == user.id or user.role == "admin"):
-        return {"text": file.meta.get("content", {}).get("text", None)}
-    else:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=ERROR_MESSAGES.NOT_FOUND,
-        )
-
-
 @router.get("/{id}/content/{file_name}", response_model=Optional[FileModel])
 async def get_file_content_by_id(id: str, user=Depends(get_verified_user)):
     file = Files.get_file_by_id(id)
 
     if file and (file.user_id == user.id or user.role == "admin"):
-        file_path = Path(file.meta["path"])
+        file_path = file.meta.get("path")
+        if file_path:
+            file_path = Path(file_path)
 
-        # Check if the file already exists in the cache
+            # Check if the file already exists in the cache
-        if file_path.is_file():
+            if file_path.is_file():
-            print(f"file_path: {file_path}")
+                print(f"file_path: {file_path}")
-            return FileResponse(file_path)
+                return FileResponse(file_path)
+            else:
+                raise HTTPException(
+                    status_code=status.HTTP_404_NOT_FOUND,
+                    detail=ERROR_MESSAGES.NOT_FOUND,
+                )
         else:
-            raise HTTPException(
+            # File path doesn’t exist, return the content as .txt if possible
-                status_code=status.HTTP_404_NOT_FOUND,
+            file_content = file.content.get("content", "")
-                detail=ERROR_MESSAGES.NOT_FOUND,
+            file_name = file.filename
+
+            # Create a generator that encodes the file content
+            def generator():
+                yield file_content.encode("utf-8")
+
+            return StreamingResponse(
+                generator(),
+                media_type="text/plain",
+                headers={"Content-Disposition": f"attachment; filename={file_name}"},
             )
     else:
         raise HTTPException(

backend/open_webui/apps/webui/routers/knowledge.py

@@ -0,0 +1,320 @@
+import json
+from typing import Optional, Union
+from pydantic import BaseModel
+from fastapi import APIRouter, Depends, HTTPException, status
+
+
+from open_webui.apps.webui.models.knowledge import (
+    Knowledges,
+    KnowledgeUpdateForm,
+    KnowledgeForm,
+    KnowledgeResponse,
+)
+from open_webui.apps.webui.models.files import Files, FileModel
+from open_webui.apps.retrieval.vector.connector import VECTOR_DB_CLIENT
+from open_webui.apps.retrieval.main import process_file, ProcessFileForm
+
+
+from open_webui.constants import ERROR_MESSAGES
+from open_webui.utils.utils import get_admin_user, get_verified_user
+
+
+router = APIRouter()
+
+############################
+# GetKnowledgeItems
+############################
+
+
+@router.get(
+    "/", response_model=Optional[Union[list[KnowledgeResponse], KnowledgeResponse]]
+)
+async def get_knowledge_items(
+    id: Optional[str] = None, user=Depends(get_verified_user)
+):
+    if id:
+        knowledge = Knowledges.get_knowledge_by_id(id=id)
+
+        if knowledge:
+            return knowledge
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail=ERROR_MESSAGES.NOT_FOUND,
+            )
+    else:
+        return [
+            KnowledgeResponse(**knowledge.model_dump())
+            for knowledge in Knowledges.get_knowledge_items()
+        ]
+
+
+############################
+# CreateNewKnowledge
+############################
+
+
+@router.post("/create", response_model=Optional[KnowledgeResponse])
+async def create_new_knowledge(form_data: KnowledgeForm, user=Depends(get_admin_user)):
+    knowledge = Knowledges.insert_new_knowledge(user.id, form_data)
+
+    if knowledge:
+        return knowledge
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.FILE_EXISTS,
+        )
+
+
+############################
+# GetKnowledgeById
+############################
+
+
+class KnowledgeFilesResponse(KnowledgeResponse):
+    files: list[FileModel]
+
+
+@router.get("/{id}", response_model=Optional[KnowledgeFilesResponse])
+async def get_knowledge_by_id(id: str, user=Depends(get_verified_user)):
+    knowledge = Knowledges.get_knowledge_by_id(id=id)
+
+    if knowledge:
+        file_ids = knowledge.data.get("file_ids", []) if knowledge.data else []
+        files = Files.get_files_by_ids(file_ids)
+
+        return KnowledgeFilesResponse(
+            **knowledge.model_dump(),
+            files=files,
+        )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# UpdateKnowledgeById
+############################
+
+
+@router.post("/{id}/update", response_model=Optional[KnowledgeFilesResponse])
+async def update_knowledge_by_id(
+    id: str,
+    form_data: KnowledgeUpdateForm,
+    user=Depends(get_admin_user),
+):
+    knowledge = Knowledges.update_knowledge_by_id(id=id, form_data=form_data)
+
+    if knowledge:
+        file_ids = knowledge.data.get("file_ids", []) if knowledge.data else []
+        files = Files.get_files_by_ids(file_ids)
+
+        return KnowledgeFilesResponse(
+            **knowledge.model_dump(),
+            files=files,
+        )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.ID_TAKEN,
+        )
+
+
+############################
+# AddFileToKnowledge
+############################
+
+
+class KnowledgeFileIdForm(BaseModel):
+    file_id: str
+
+
+@router.post("/{id}/file/add", response_model=Optional[KnowledgeFilesResponse])
+def add_file_to_knowledge_by_id(
+    id: str,
+    form_data: KnowledgeFileIdForm,
+    user=Depends(get_admin_user),
+):
+    knowledge = Knowledges.get_knowledge_by_id(id=id)
+    file = Files.get_file_by_id(form_data.file_id)
+    if not file:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+    if not file.data:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.FILE_NOT_PROCESSED,
+        )
+
+    # Add content to the vector database
+    try:
+        process_file(ProcessFileForm(file_id=form_data.file_id, collection_name=id))
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        )
+
+    if knowledge:
+        data = knowledge.data or {}
+        file_ids = data.get("file_ids", [])
+
+        if form_data.file_id not in file_ids:
+            file_ids.append(form_data.file_id)
+            data["file_ids"] = file_ids
+
+            knowledge = Knowledges.update_knowledge_by_id(
+                id=id, form_data=KnowledgeUpdateForm(data=data)
+            )
+
+            if knowledge:
+                files = Files.get_files_by_ids(file_ids)
+
+                return KnowledgeFilesResponse(
+                    **knowledge.model_dump(),
+                    files=files,
+                )
+            else:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=ERROR_MESSAGES.DEFAULT("knowledge"),
+                )
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("file_id"),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+@router.post("/{id}/file/update", response_model=Optional[KnowledgeFilesResponse])
+def update_file_from_knowledge_by_id(
+    id: str,
+    form_data: KnowledgeFileIdForm,
+    user=Depends(get_admin_user),
+):
+    knowledge = Knowledges.get_knowledge_by_id(id=id)
+    file = Files.get_file_by_id(form_data.file_id)
+    if not file:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+    # Remove content from the vector database
+    VECTOR_DB_CLIENT.delete(
+        collection_name=knowledge.id, filter={"file_id": form_data.file_id}
+    )
+
+    # Add content to the vector database
+    try:
+        process_file(ProcessFileForm(file_id=form_data.file_id, collection_name=id))
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        )
+
+    if knowledge:
+        data = knowledge.data or {}
+        file_ids = data.get("file_ids", [])
+
+        files = Files.get_files_by_ids(file_ids)
+
+        return KnowledgeFilesResponse(
+            **knowledge.model_dump(),
+            files=files,
+        )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# RemoveFileFromKnowledge
+############################
+
+
+@router.post("/{id}/file/remove", response_model=Optional[KnowledgeFilesResponse])
+def remove_file_from_knowledge_by_id(
+    id: str,
+    form_data: KnowledgeFileIdForm,
+    user=Depends(get_admin_user),
+):
+    knowledge = Knowledges.get_knowledge_by_id(id=id)
+    file = Files.get_file_by_id(form_data.file_id)
+    if not file:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+    # Remove content from the vector database
+    VECTOR_DB_CLIENT.delete(
+        collection_name=knowledge.id, filter={"file_id": form_data.file_id}
+    )
+
+    result = VECTOR_DB_CLIENT.query(
+        collection_name=knowledge.id,
+        filter={"file_id": form_data.file_id},
+    )
+
+    Files.delete_file_by_id(form_data.file_id)
+
+    if knowledge:
+        data = knowledge.data or {}
+        file_ids = data.get("file_ids", [])
+
+        if form_data.file_id in file_ids:
+            file_ids.remove(form_data.file_id)
+            data["file_ids"] = file_ids
+
+            knowledge = Knowledges.update_knowledge_by_id(
+                id=id, form_data=KnowledgeUpdateForm(data=data)
+            )
+
+            if knowledge:
+                files = Files.get_files_by_ids(file_ids)
+
+                return KnowledgeFilesResponse(
+                    **knowledge.model_dump(),
+                    files=files,
+                )
+            else:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=ERROR_MESSAGES.DEFAULT("knowledge"),
+                )
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("file_id"),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# DeleteKnowledgeById
+############################
+
+
+@router.delete("/{id}/delete", response_model=bool)
+async def delete_knowledge_by_id(id: str, user=Depends(get_admin_user)):
+    VECTOR_DB_CLIENT.delete_collection(collection_name=id)
+    result = Knowledges.delete_knowledge_by_id(id=id)
+    return result

backend/open_webui/config.py

@@ -56,9 +56,6 @@ def run_migrations():
         print(f"Error: {e}")
 
 
-run_migrations()
-
-
 class Config(Base):
     __tablename__ = "config"
 

backend/open_webui/constants.py

@@ -34,8 +34,8 @@ class ERROR_MESSAGES(str, Enum):
 
     ID_TAKEN = "Uh-oh! This id is already registered. Please choose another id string."
     MODEL_ID_TAKEN = "Uh-oh! This model id is already registered. Please choose another model id string."
-
     NAME_TAG_TAKEN = "Uh-oh! This name tag is already registered. Please choose another name tag string."
+
     INVALID_TOKEN = (
         "Your session has expired or the token is invalid. Please sign in again."
     )
@@ -94,6 +94,11 @@ class ERROR_MESSAGES(str, Enum):
         lambda size="": f"Oops! The file you're trying to upload is too large. Please upload a file that is less than {size}."
     )
 
+    DUPLICATE_CONTENT = (
+        "Duplicate content detected. Please provide unique content to proceed."
+    )
+    FILE_NOT_PROCESSED = "Extracted content is not available for this file. Please ensure that the file is processed before proceeding."
+
 
 class TASKS(str, Enum):
     def __str__(self) -> str:

backend/open_webui/migrations/scripts/revision.py

@@ -1,19 +0,0 @@
-from alembic import command
-from alembic.config import Config
-
-from open_webui.env import OPEN_WEBUI_DIR
-
-alembic_cfg = Config(OPEN_WEBUI_DIR / "alembic.ini")
-
-# Set the script location dynamically
-migrations_path = OPEN_WEBUI_DIR / "migrations"
-alembic_cfg.set_main_option("script_location", str(migrations_path))
-
-
-def revision(message: str) -> None:
-    command.revision(alembic_cfg, message=message, autogenerate=False)
-
-
-if __name__ == "__main__":
-    input_message = input("Enter the revision message: ")
-    revision(input_message)

backend/open_webui/migrations/util.py

@@ -7,3 +7,9 @@ def get_existing_tables():
     inspector = Inspector.from_engine(con)
     tables = set(inspector.get_table_names())
     return tables
+
+
+def get_revision_id():
+    import uuid
+
+    return str(uuid.uuid4()).replace("-", "")[:12]

backend/open_webui/migrations/versions/6a39f3d8e55c_add_knowledge_table.py

@@ -0,0 +1,80 @@
+"""Add knowledge table
+
+Revision ID: 6a39f3d8e55c
+Revises: c0fbf31ca0db
+Create Date: 2024-10-01 14:02:35.241684
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import table, column, select
+import json
+
+
+revision = "6a39f3d8e55c"
+down_revision = "c0fbf31ca0db"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Creating the 'knowledge' table
+    print("Creating knowledge table")
+    knowledge_table = op.create_table(
+        "knowledge",
+        sa.Column("id", sa.Text(), primary_key=True),
+        sa.Column("user_id", sa.Text(), nullable=False),
+        sa.Column("name", sa.Text(), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("data", sa.JSON(), nullable=True),
+        sa.Column("meta", sa.JSON(), nullable=True),
+        sa.Column("created_at", sa.BigInteger(), nullable=False),
+        sa.Column("updated_at", sa.BigInteger(), nullable=True),
+    )
+
+    print("Migrating data from document table to knowledge table")
+    # Representation of the existing 'document' table
+    document_table = table(
+        "document",
+        column("collection_name", sa.String()),
+        column("user_id", sa.String()),
+        column("name", sa.String()),
+        column("title", sa.Text()),
+        column("content", sa.Text()),
+        column("timestamp", sa.BigInteger()),
+    )
+
+    # Select all from existing document table
+    documents = op.get_bind().execute(
+        select(
+            document_table.c.collection_name,
+            document_table.c.user_id,
+            document_table.c.name,
+            document_table.c.title,
+            document_table.c.content,
+            document_table.c.timestamp,
+        )
+    )
+
+    # Insert data into knowledge table from document table
+    for doc in documents:
+        op.get_bind().execute(
+            knowledge_table.insert().values(
+                id=doc.collection_name,
+                user_id=doc.user_id,
+                description=doc.name,
+                meta={
+                    "legacy": True,
+                    "document": True,
+                    "tags": json.loads(doc.content or "{}").get("tags", []),
+                },
+                name=doc.title,
+                created_at=doc.timestamp,
+                updated_at=doc.timestamp,  # using created_at for both created_at and updated_at in project
+            )
+        )
+
+
+def downgrade():
+    op.drop_table("knowledge")

backend/open_webui/migrations/versions/c0fbf31ca0db_update_file_table.py

@@ -0,0 +1,32 @@
+"""Update file table
+
+Revision ID: c0fbf31ca0db
+Revises: ca81bd47c050
+Create Date: 2024-09-20 15:26:35.241684
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "c0fbf31ca0db"
+down_revision: Union[str, None] = "ca81bd47c050"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("file", sa.Column("hash", sa.Text(), nullable=True))
+    op.add_column("file", sa.Column("data", sa.JSON(), nullable=True))
+    op.add_column("file", sa.Column("updated_at", sa.BigInteger(), nullable=True))
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("file", "updated_at")
+    op.drop_column("file", "data")
+    op.drop_column("file", "hash")

src/lib/apis/files/index.ts

@@ -92,6 +92,40 @@ export const getFileById = async (token: string, id: string) => {
 	return res;
 };
 
+export const updateFileDataContentById = async (token: string, id: string, content: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/files/${id}/data/content/update`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			content: content
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 export const getFileContentById = async (id: string) => {
 	let error = null;
 

src/lib/apis/knowledge/index.ts

@@ -0,0 +1,276 @@
+import { WEBUI_API_BASE_URL } from '$lib/constants';
+
+export const createNewKnowledge = async (token: string, name: string, description: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/create`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			name: name,
+			description: description
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const getKnowledgeItems = async (token: string = '') => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const getKnowledgeById = async (token: string, id: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+type KnowledgeUpdateForm = {
+	name?: string;
+	description?: string;
+	data?: object;
+};
+
+export const updateKnowledgeById = async (token: string, id: string, form: KnowledgeUpdateForm) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/update`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			name: form?.name ? form.name : undefined,
+			description: form?.description ? form.description : undefined,
+			data: form?.data ? form.data : undefined
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const addFileToKnowledgeById = async (token: string, id: string, fileId: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/file/add`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			file_id: fileId
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const updateFileFromKnowledgeById = async (token: string, id: string, fileId: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/file/update`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			file_id: fileId
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const removeFileFromKnowledgeById = async (token: string, id: string, fileId: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/file/remove`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			file_id: fileId
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const deleteKnowledgeById = async (token: string, id: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/delete`, {
+		method: 'DELETE',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};

src/lib/apis/retrieval/index.ts

@@ -306,7 +306,11 @@ export interface SearchDocument {
 	filenames: string[];
 }
 
-export const processFile = async (token: string, file_id: string) => {
+export const processFile = async (
+	token: string,
+	file_id: string,
+	collection_name: string | null = null
+) => {
 	let error = null;
 
 	const res = await fetch(`${RAG_API_BASE_URL}/process/file`, {
@@ -317,7 +321,8 @@ export const processFile = async (token: string, file_id: string) => {
 			authorization: `Bearer ${token}`
 		},
 		body: JSON.stringify({
-			file_id: file_id
+			file_id: file_id,
+			collection_name: collection_name ? collection_name : undefined
 		})
 	})
 		.then(async (res) => {

src/lib/components/AddFilesPlaceholder.svelte

@@ -1,10 +1,18 @@
 <script>
 	import { getContext } from 'svelte';
+
+	export let title = '';
 	const i18n = getContext('i18n');
 </script>
 
 <div class="  text-center text-6xl mb-3">📄</div>
-<div class="text-center dark:text-white text-2xl font-semibold z-50">{$i18n.t('Add Files')}</div>
+<div class="text-center dark:text-white text-2xl font-semibold z-50">
+	{#if title}
+		{title}
+	{:else}
+		{$i18n.t('Add Files')}
+	{/if}
+</div>
 
 <slot
 	><div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">

src/lib/components/admin/Settings/Documents.svelte

@@ -1,10 +1,10 @@
 <script lang="ts">
+	import { toast } from 'svelte-sonner';
+
 	import { onMount, getContext, createEventDispatcher } from 'svelte';
 
 	const dispatch = createEventDispatcher();
 
-	import { getDocs } from '$lib/apis/documents';
-	import { deleteAllFiles, deleteFileById } from '$lib/apis/files';
 	import {
 		getQuerySettings,
 		processDocsDir,
@@ -18,11 +18,13 @@
 		getRAGConfig,
 		updateRAGConfig
 	} from '$lib/apis/retrieval';
+
+	import { knowledge, models } from '$lib/stores';
+	import { getKnowledgeItems } from '$lib/apis/knowledge';
+	import { deleteAllFiles, deleteFileById } from '$lib/apis/files';
+
 	import ResetUploadDirConfirmDialog from '$lib/components/common/ConfirmDialog.svelte';
 	import ResetVectorDBConfirmDialog from '$lib/components/common/ConfirmDialog.svelte';
-
-	import { documents, models } from '$lib/stores';
-	import { toast } from 'svelte-sonner';
 	import SensitiveInput from '$lib/components/common/SensitiveInput.svelte';
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
 
@@ -67,7 +69,7 @@
 		scanDirLoading = false;
 
 		if (res) {
-			await documents.set(await getDocs(localStorage.token));
+			await knowledge.set(await getKnowledgeItems(localStorage.token));
 			toast.success($i18n.t('Scan complete!'));
 		}
 	};

src/lib/components/admin/Settings/WebSearch.svelte

@@ -2,7 +2,7 @@
 	import { getRAGConfig, updateRAGConfig } from '$lib/apis/retrieval';
 	import Switch from '$lib/components/common/Switch.svelte';
 
-	import { documents, models } from '$lib/stores';
+	import { models } from '$lib/stores';
 	import { onMount, getContext } from 'svelte';
 	import { toast } from 'svelte-sonner';
 	import SensitiveInput from '$lib/components/common/SensitiveInput.svelte';

src/lib/components/chat/Controls/Controls.svelte

@@ -35,7 +35,7 @@
 					{#each chatFiles as file, fileIdx}
 						<FileItem
 							className="w-full"
-							{file}
+							item={file}
 							edit={true}
 							url={`${file?.url}`}
 							name={file.name}

src/lib/components/chat/MessageInput.svelte

@@ -125,16 +125,17 @@
 		}
 
 		try {
+			// During the file upload, file content is automatically extracted.
 			const uploadedFile = await uploadFile(localStorage.token, file);
 
 			if (uploadedFile) {
-				fileItem.status = 'uploaded';
+				fileItem.status = 'processed';
 				fileItem.file = uploadedFile;
 				fileItem.id = uploadedFile.id;
+				fileItem.collection_name = uploadedFile?.meta?.collection_name;
 				fileItem.url = `${WEBUI_API_BASE_URL}/files/${uploadedFile.id}`;
 
-				// Try to extract content of the file for retrieval, even non-supported file types
+				files = files;
-				processFileItem(fileItem);
 			} else {
 				files = files.filter((item) => item.status !== null);
 			}
@@ -144,26 +145,6 @@
 		}
 	};
 
-	const processFileItem = async (fileItem) => {
-		try {
-			const res = await processFile(localStorage.token, fileItem.id);
-			if (res) {
-				fileItem.status = 'processed';
-				fileItem.collection_name = res.collection_name;
-				fileItem.file = {
-					...fileItem.file,
-					content: res.content
-				};
-
-				files = files;
-			}
-		} catch (e) {
-			// We keep the file in the files list even if it fails to process
-			fileItem.status = 'processed';
-			files = files;
-		}
-	};
-
 	const inputFilesHandler = async (inputFiles) => {
 		inputFiles.forEach((file) => {
 			console.log(file, file.name.split('.').at(-1));
@@ -456,7 +437,7 @@
 											</div>
 										{:else}
 											<FileItem
-												{file}
+												item={file}
 												name={file.name}
 												type={file.type}
 												size={file?.size}

src/lib/components/chat/MessageInput/Commands.svelte

@@ -5,7 +5,7 @@
 	const dispatch = createEventDispatcher();
 
 	import Prompts from './Commands/Prompts.svelte';
-	import Documents from './Commands/Documents.svelte';
+	import Knowledge from './Commands/Knowledge.svelte';
 	import Models from './Commands/Models.svelte';
 
 	import { removeLastWordFromString } from '$lib/utils';
@@ -97,7 +97,7 @@
 	{#if command?.charAt(0) === '/'}
 		<Prompts bind:this={commandElement} bind:prompt bind:files {command} />
 	{:else if command?.charAt(0) === '#'}
-		<Documents
+		<Knowledge
 			bind:this={commandElement}
 			bind:prompt
 			{command}
@@ -114,7 +114,7 @@
 				files = [
 					...files,
 					{
-						type: e?.detail?.type ?? 'file',
+						type: e?.detail?.meta?.document ? 'file' : 'collection',
 						...e.detail,
 						status: 'processed'
 					}

src/lib/components/chat/MessageInput/Commands/Knowledge.svelte

@@ -1,10 +1,10 @@
 <script lang="ts">
-	import { createEventDispatcher } from 'svelte';
-
-	import { documents } from '$lib/stores';
-	import { removeLastWordFromString, isValidHttpUrl } from '$lib/utils';
-	import { tick, getContext } from 'svelte';
 	import { toast } from 'svelte-sonner';
+	import Fuse from 'fuse.js';
+
+	import { createEventDispatcher, tick, getContext, onMount } from 'svelte';
+	import { removeLastWordFromString, isValidHttpUrl } from '$lib/utils';
+	import { knowledge } from '$lib/stores';
 
 	const i18n = getContext('i18n');
 
@@ -14,60 +14,22 @@
 	const dispatch = createEventDispatcher();
 	let selectedIdx = 0;
 
+	let items = [];
+	let fuse = null;
+
 	let filteredItems = [];
-	let filteredDocs = [];
+	$: if (fuse) {
-
+		filteredItems = command.slice(1)
-	let collections = [];
+			? fuse.search(command).map((e) => {
-
+					return e.item;
-	$: collections = [
+				})
-		...($documents.length > 0
+			: items;
-			? [
+	}
-					{
-						name: 'All Documents',
-						type: 'collection',
-						title: $i18n.t('All Documents'),
-						collection_names: $documents.map((doc) => doc.collection_name)
-					}
-				]
-			: []),
-		...$documents
-			.reduce((a, e, i, arr) => {
-				return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])];
-			}, [])
-			.map((tag) => ({
-				name: tag,
-				type: 'collection',
-				collection_names: $documents
-					.filter((doc) => (doc?.content?.tags ?? []).map((tag) => tag.name).includes(tag))
-					.map((doc) => doc.collection_name)
-			}))
-	];
-
-	$: filteredCollections = collections
-		.filter((collection) => findByName(collection, command))
-		.sort((a, b) => a.name.localeCompare(b.name));
-
-	$: filteredDocs = $documents
-		.filter((doc) => findByName(doc, command))
-		.sort((a, b) => a.title.localeCompare(b.title));
-
-	$: filteredItems = [...filteredCollections, ...filteredDocs];
 
 	$: if (command) {
 		selectedIdx = 0;
-
-		console.log(filteredCollections);
 	}
 
-	type ObjectWithName = {
-		name: string;
-	};
-
-	const findByName = (obj: ObjectWithName, command: string) => {
-		const name = obj.name.toLowerCase();
-		return name.includes(command.toLowerCase().split(' ')?.at(0)?.substring(1) ?? '');
-	};
-
 	export const selectUp = () => {
 		selectedIdx = Math.max(0, selectedIdx - 1);
 	};
@@ -76,8 +38,8 @@
 		selectedIdx = Math.min(selectedIdx + 1, filteredItems.length - 1);
 	};
 
-	const confirmSelect = async (doc) => {
+	const confirmSelect = async (item) => {
-		dispatch('select', doc);
+		dispatch('select', item);
 
 		prompt = removeLastWordFromString(prompt, command);
 		const chatInputElement = document.getElementById('chat-textarea');
@@ -108,6 +70,48 @@
 		chatInputElement?.focus();
 		await tick();
 	};
+
+	onMount(() => {
+		let legacy_documents = $knowledge.filter((item) => item?.meta?.document);
+		let legacy_collections =
+			legacy_documents.length > 0
+				? [
+						{
+							name: 'All Documents',
+							legacy: true,
+							type: 'collection',
+							description: 'Deprecated (legacy collection), please create a new knowledge base.',
+							title: $i18n.t('All Documents'),
+							collection_names: legacy_documents.map((item) => item.id)
+						},
+
+						...legacy_documents
+							.reduce((a, item) => {
+								return [...new Set([...a, ...(item?.meta?.tags ?? []).map((tag) => tag.name)])];
+							}, [])
+							.map((tag) => ({
+								name: tag,
+								legacy: true,
+								type: 'collection',
+								description: 'Deprecated (legacy collection), please create a new knowledge base.',
+								collection_names: legacy_documents
+									.filter((item) => (item?.meta?.tags ?? []).map((tag) => tag.name).includes(tag))
+									.map((item) => item.id)
+							}))
+					]
+				: [];
+
+		items = [...$knowledge, ...legacy_collections].map((item) => {
+			return {
+				...item,
+				...{ legacy: item?.legacy ?? item?.meta?.document ?? undefined }
+			};
+		});
+
+		fuse = new Fuse(items, {
+			keys: ['name', 'description']
+		});
+	});
 </script>
 
 {#if filteredItems.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
@@ -124,39 +128,50 @@
 				class="max-h-60 flex flex-col w-full rounded-r-xl bg-white dark:bg-gray-900 dark:text-gray-100"
 			>
 				<div class="m-1 overflow-y-auto p-1 rounded-r-xl space-y-0.5 scrollbar-hidden">
-					{#each filteredItems as doc, docIdx}
+					{#each filteredItems as item, idx}
 						<button
-							class=" px-3 py-1.5 rounded-xl w-full text-left {docIdx === selectedIdx
+							class=" px-3 py-1.5 rounded-xl w-full text-left {idx === selectedIdx
 								? ' bg-gray-50 dark:bg-gray-850 dark:text-gray-100 selected-command-option-button'
 								: ''}"
 							type="button"
 							on:click={() => {
-								console.log(doc);
+								console.log(item);
-
+								confirmSelect(item);
-								confirmSelect(doc);
 							}}
 							on:mousemove={() => {
-								selectedIdx = docIdx;
+								selectedIdx = idx;
 							}}
 							on:focus={() => {}}
 						>
-							{#if doc.type === 'collection'}
+							<div class=" font-medium text-black dark:text-gray-100 flex items-center gap-1">
-								<div class=" font-medium text-black dark:text-gray-100 line-clamp-1">
+								{#if item.legacy}
-									{doc?.title ?? `#${doc.name}`}
+									<div
-								</div>
+										class="bg-gray-500/20 text-gray-700 dark:text-gray-200 rounded uppercase text-xs font-bold px-1"
+									>
+										Legacy
+									</div>
+								{:else if item?.meta?.document}
+									<div
+										class="bg-gray-500/20 text-gray-700 dark:text-gray-200 rounded uppercase text-xs font-bold px-1"
+									>
+										Document
+									</div>
+								{:else}
+									<div
+										class="bg-green-500/20 text-green-700 dark:text-green-200 rounded uppercase text-xs font-bold px-1"
+									>
+										Collection
+									</div>
+								{/if}
 
-								<div class=" text-xs text-gray-600 dark:text-gray-100 line-clamp-1">
+								<div class="line-clamp-1">
-									{$i18n.t('Collection')}
+									{item.name}
-								</div>
-							{:else}
-								<div class=" font-medium text-black dark:text-gray-100 line-clamp-1">
-									#{doc.name} ({doc.filename})
 								</div>
+							</div>
 
-								<div class=" text-xs text-gray-600 dark:text-gray-100 line-clamp-1">
+							<div class=" text-xs text-gray-600 dark:text-gray-100 line-clamp-1">
-									{doc.title}
+								{item?.description}
-								</div>
+							</div>
-							{/if}
 						</button>
 					{/each}
 

src/lib/components/chat/Messages/UserMessage.svelte

@@ -127,7 +127,7 @@
 								<img src={file.url} alt="input" class=" max-h-96 rounded-lg" draggable="false" />
 							{:else}
 								<FileItem
-									{file}
+									item={file}
 									url={file.url}
 									name={file.name}
 									type={file.type}

src/lib/components/common/Badge.svelte

@@ -0,0 +1,18 @@
+<script lang="ts">
+	export let type = 'info';
+	export let content = '';
+
+	const classNames: Record<string, string> = {
+		info: 'bg-blue-500/20 text-blue-700 dark:text-blue-200 ',
+		success: 'bg-green-500/20 text-green-700 dark:text-green-200',
+		warning: 'bg-yellow-500/20 text-yellow-700 dark:text-yellow-200',
+		error: 'bg-red-500/20 text-red-700 dark:text-red-200'
+	};
+</script>
+
+<div
+	class=" text-xs font-bold {classNames[type] ??
+		classNames['info']}  w-fit px-2 rounded uppercase line-clamp-1 mr-0.5"
+>
+	{content}
+</div>

src/lib/components/common/FileItem.svelte

@@ -3,18 +3,19 @@
 	import { formatFileSize } from '$lib/utils';
 
 	import FileItemModal from './FileItemModal.svelte';
+	import GarbageBin from '../icons/GarbageBin.svelte';
 
 	const i18n = getContext('i18n');
 	const dispatch = createEventDispatcher();
 
-	export let className = 'w-72';
+	export let className = 'w-60';
-	export let colorClassName = 'bg-white dark:bg-gray-800';
+	export let colorClassName = 'bg-white dark:bg-gray-850 border border-gray-50 dark:border-white/5';
 	export let url: string | null = null;
 
 	export let dismissible = false;
 	export let status = 'processed';
 
-	export let file = null;
+	export let item = null;
 	export let edit = false;
 
 	export let name: string;
@@ -24,115 +25,113 @@
 	let showModal = false;
 </script>
 
-{#if file}
+{#if item}
-	<FileItemModal bind:show={showModal} bind:file {edit} />
+	<FileItemModal bind:show={showModal} bind:item {edit} />
 {/if}
 
-<div class="relative group">
+<button
-	<button
+	class="relative group p-1.5 {className} flex items-center {colorClassName} rounded-2xl text-left"
-		class="h-14 {className} flex items-center space-x-3 {colorClassName} rounded-xl border border-gray-100 dark:border-gray-800 text-left"
+	type="button"
-		type="button"
+	on:click={async () => {
-		on:click={async () => {
+		if (item?.file?.data?.content) {
-			if (file?.file?.content) {
+			showModal = !showModal;
-				showModal = !showModal;
+		} else {
-			} else {
+			if (url) {
-				if (url) {
+				if (type === 'file') {
-					if (type === 'file') {
+					window.open(`${url}/content`, '_blank').focus();
-						window.open(`${url}/content`, '_blank').focus();
+				} else {
-					} else {
+					window.open(`${url}`, '_blank').focus();
-						window.open(`${url}`, '_blank').focus();
-					}
 				}
 			}
+		}
 
-			dispatch('click');
+		dispatch('click');
-		}}
+	}}
-	>
+>
-		<div class="p-4 py-[1.1rem] bg-red-400 text-white rounded-l-xl">
+	<div class="p-3 bg-black/20 dark:bg-white/10 text-white rounded-xl">
-			{#if status === 'processed'}
+		{#if status === 'processed'}
-				<svg
+			<svg
-					xmlns="http://www.w3.org/2000/svg"
+				xmlns="http://www.w3.org/2000/svg"
-					viewBox="0 0 24 24"
+				viewBox="0 0 24 24"
-					fill="currentColor"
+				fill="currentColor"
-					class=" size-5"
+				class=" size-5"
-				>
+			>
-					<path
+				<path
-						fill-rule="evenodd"
+					fill-rule="evenodd"
-						d="M5.625 1.5c-1.036 0-1.875.84-1.875 1.875v17.25c0 1.035.84 1.875 1.875 1.875h12.75c1.035 0 1.875-.84 1.875-1.875V12.75A3.75 3.75 0 0 0 16.5 9h-1.875a1.875 1.875 0 0 1-1.875-1.875V5.25A3.75 3.75 0 0 0 9 1.5H5.625ZM7.5 15a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 7.5 15Zm.75 2.25a.75.75 0 0 0 0 1.5H12a.75.75 0 0 0 0-1.5H8.25Z"
+					d="M5.625 1.5c-1.036 0-1.875.84-1.875 1.875v17.25c0 1.035.84 1.875 1.875 1.875h12.75c1.035 0 1.875-.84 1.875-1.875V12.75A3.75 3.75 0 0 0 16.5 9h-1.875a1.875 1.875 0 0 1-1.875-1.875V5.25A3.75 3.75 0 0 0 9 1.5H5.625ZM7.5 15a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 7.5 15Zm.75 2.25a.75.75 0 0 0 0 1.5H12a.75.75 0 0 0 0-1.5H8.25Z"
-						clip-rule="evenodd"
+					clip-rule="evenodd"
-					/>
+				/>
-					<path
+				<path
-						d="M12.971 1.816A5.23 5.23 0 0 1 14.25 5.25v1.875c0 .207.168.375.375.375H16.5a5.23 5.23 0 0 1 3.434 1.279 9.768 9.768 0 0 0-6.963-6.963Z"
+					d="M12.971 1.816A5.23 5.23 0 0 1 14.25 5.25v1.875c0 .207.168.375.375.375H16.5a5.23 5.23 0 0 1 3.434 1.279 9.768 9.768 0 0 0-6.963-6.963Z"
-					/>
+				/>
-				</svg>
+			</svg>
+		{:else}
+			<svg
+				class=" size-5 translate-y-[0.5px]"
+				fill="currentColor"
+				viewBox="0 0 24 24"
+				xmlns="http://www.w3.org/2000/svg"
+				><style>
+					.spinner_qM83 {
+						animation: spinner_8HQG 1.05s infinite;
+					}
+					.spinner_oXPr {
+						animation-delay: 0.1s;
+					}
+					.spinner_ZTLf {
+						animation-delay: 0.2s;
+					}
+					@keyframes spinner_8HQG {
+						0%,
+						57.14% {
+							animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
+							transform: translate(0);
+						}
+						28.57% {
+							animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
+							transform: translateY(-6px);
+						}
+						100% {
+							transform: translate(0);
+						}
+					}
+				</style><circle class="spinner_qM83" cx="4" cy="12" r="2.5" /><circle
+					class="spinner_qM83 spinner_oXPr"
+					cx="12"
+					cy="12"
+					r="2.5"
+				/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="2.5" /></svg
+			>
+		{/if}
+	</div>
+
+	<div class="flex flex-col justify-center -space-y-0.5 ml-1 px-2.5 w-full">
+		<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1 mb-1">
+			{name}
+		</div>
+
+		<div class=" flex justify-between text-gray-500 text-xs">
+			{#if type === 'file'}
+				{$i18n.t('File')}
+			{:else if type === 'doc'}
+				{$i18n.t('Document')}
+			{:else if type === 'collection'}
+				{$i18n.t('Collection')}
 			{:else}
-				<svg
+				<span class=" capitalize">{type}</span>
-					class=" size-5 translate-y-[0.5px]"
+			{/if}
-					fill="currentColor"
+			{#if size}
-					viewBox="0 0 24 24"
+				<span class="capitalize">{formatFileSize(size)}</span>
-					xmlns="http://www.w3.org/2000/svg"
-					><style>
-						.spinner_qM83 {
-							animation: spinner_8HQG 1.05s infinite;
-						}
-						.spinner_oXPr {
-							animation-delay: 0.1s;
-						}
-						.spinner_ZTLf {
-							animation-delay: 0.2s;
-						}
-						@keyframes spinner_8HQG {
-							0%,
-							57.14% {
-								animation-timing-function: cubic-bezier(0.33, 0.66, 0.66, 1);
-								transform: translate(0);
-							}
-							28.57% {
-								animation-timing-function: cubic-bezier(0.33, 0, 0.66, 0.33);
-								transform: translateY(-6px);
-							}
-							100% {
-								transform: translate(0);
-							}
-						}
-					</style><circle class="spinner_qM83" cx="4" cy="12" r="2.5" /><circle
-						class="spinner_qM83 spinner_oXPr"
-						cx="12"
-						cy="12"
-						r="2.5"
-					/><circle class="spinner_qM83 spinner_ZTLf" cx="20" cy="12" r="2.5" /></svg
-				>
 			{/if}
 		</div>
-
+	</div>
-		<div class="flex flex-col justify-center -space-y-0.5 pl-1.5 pr-4 w-full">
-			<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1 mb-1">
-				{name}
-			</div>
-
-			<div class=" flex justify-between text-gray-500 text-xs">
-				{#if type === 'file'}
-					{$i18n.t('File')}
-				{:else if type === 'doc'}
-					{$i18n.t('Document')}
-				{:else if type === 'collection'}
-					{$i18n.t('Collection')}
-				{:else}
-					<span class=" capitalize">{type}</span>
-				{/if}
-				{#if size}
-					<span class="capitalize">{formatFileSize(size)}</span>
-				{/if}
-			</div>
-		</div>
-	</button>
 
 	{#if dismissible}
 		<div class=" absolute -top-1 -right-1">
 			<button
 				class=" bg-gray-400 text-white border border-white rounded-full group-hover:visible invisible transition"
 				type="button"
-				on:click={() => {
+				on:click|stopPropagation={() => {
 					dispatch('dismiss');
 				}}
 			>
@@ -147,6 +146,15 @@
 					/>
 				</svg>
 			</button>
+
+			<!-- <button
+				class=" p-1 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-full group-hover:visible invisible transition"
+				type="button"
+				on:click={() => {
+				}}
+			>
+				<GarbageBin />
+			</button> -->
 		</div>
 	{/if}
-</div>
+</button>

src/lib/components/common/FileItemModal.svelte

@@ -10,7 +10,7 @@
 	import Switch from './Switch.svelte';
 	import Tooltip from './Tooltip.svelte';
 
-	export let file;
+	export let item;
 	export let show = false;
 
 	export let edit = false;
@@ -18,9 +18,9 @@
 	let enableFullContent = false;
 
 	onMount(() => {
-		console.log(file);
+		console.log(item);
 
-		if (file?.context === 'full') {
+		if (item?.context === 'full') {
 			enableFullContent = true;
 		}
 	});
@@ -33,11 +33,11 @@
 				<div>
 					<div class=" font-medium text-lg dark:text-gray-100">
 						<a
-							href={file.url ? (file.type === 'file' ? `${file.url}/content` : `${file.url}`) : '#'}
+							href={item.url ? (item.type === 'file' ? `${item.url}/content` : `${item.url}`) : '#'}
 							target="_blank"
 							class="hover:underline line-clamp-1"
 						>
-							{file?.name ?? 'File'}
+							{item?.name ?? 'File'}
 						</a>
 					</div>
 				</div>
@@ -56,14 +56,14 @@
 			<div>
 				<div class="flex flex-col items-center md:flex-row gap-1 justify-between w-full">
 					<div class=" flex flex-wrap text-sm gap-1 text-gray-500">
-						{#if file.size}
+						{#if item.size}
-							<div class="capitalize shrink-0">{formatFileSize(file.size)}</div>
+							<div class="capitalize shrink-0">{formatFileSize(item.size)}</div>
 							•
 						{/if}
 
-						{#if file?.file?.content}
+						{#if item?.file?.data?.content}
 							<div class="capitalize shrink-0">
-								{getLineCount(file?.file?.content ?? '')} extracted lines
+								{getLineCount(item?.file?.data?.content ?? '')} extracted lines
 							</div>
 
 							<div class="flex items-center gap-1 shrink-0">
@@ -90,7 +90,7 @@
 									<Switch
 										bind:state={enableFullContent}
 										on:change={(e) => {
-											file.context = e.detail ? 'full' : undefined;
+											item.context = e.detail ? 'full' : undefined;
 										}}
 									/>
 								</div>
@@ -102,7 +102,7 @@
 		</div>
 
 		<div class="max-h-96 overflow-scroll scrollbar-hidden text-xs whitespace-pre-wrap">
-			{file?.file?.content ?? 'No content'}
+			{item?.file?.data?.content ?? 'No content'}
 		</div>
 	</div>
 </Modal>

src/lib/components/icons/BarsArrowUp.svelte

@@ -0,0 +1,19 @@
+<script lang="ts">
+	export let className = 'size-4';
+	export let strokeWidth = '1.5';
+</script>
+
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	fill="none"
+	viewBox="0 0 24 24"
+	stroke-width={strokeWidth}
+	stroke="currentColor"
+	class={className}
+>
+	<path
+		stroke-linecap="round"
+		stroke-linejoin="round"
+		d="M3 4.5h14.25M3 9h9.75M3 13.5h5.25m5.25-.75L17.25 9m0 0L21 12.75M17.25 9v12"
+	/>
+</svg>

src/lib/components/icons/BookOpen.svelte

@@ -0,0 +1,19 @@
+<script lang="ts">
+	export let className = 'size-4';
+	export let strokeWidth = '2';
+</script>
+
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	fill="none"
+	viewBox="0 0 24 24"
+	stroke-width={strokeWidth}
+	stroke="currentColor"
+	class={className}
+>
+	<path
+		stroke-linecap="round"
+		stroke-linejoin="round"
+		d="M12 6.042A8.967 8.967 0 0 0 6 3.75c-1.052 0-2.062.18-3 .512v14.25A8.987 8.987 0 0 1 6 18c2.305 0 4.408.867 6 2.292m0-14.25a8.966 8.966 0 0 1 6-2.292c1.052 0 2.062.18 3 .512v14.25A8.987 8.987 0 0 0 18 18a8.967 8.967 0 0 0-6 2.292m0-14.25v14.25"
+	/>
+</svg>

src/lib/components/icons/FloppyDisk.svelte

@@ -0,0 +1,20 @@
+<script lang="ts">
+	export let className = 'size-4';
+	export let strokeWidth = '1.5';
+</script>
+
+<svg
+	stroke="currentColor"
+	class={className}
+	aria-hidden="true"
+	xmlns="http://www.w3.org/2000/svg"
+	fill="none"
+	viewBox="0 0 24 24"
+>
+	<path
+		stroke="currentColor"
+		stroke-linecap="round"
+		stroke-width={strokeWidth}
+		d="M11 16h2m6.707-9.293-2.414-2.414A1 1 0 0 0 16.586 4H5a1 1 0 0 0-1 1v14a1 1 0 0 0 1 1h14a1 1 0 0 0 1-1V7.414a1 1 0 0 0-.293-.707ZM16 20v-6a1 1 0 0 0-1-1H9a1 1 0 0 0-1 1v6h8ZM9 4h6v3a1 1 0 0 1-1 1h-4a1 1 0 0 1-1-1V4Z"
+	/>
+</svg>

src/lib/components/workspace/Knowledge.svelte

@@ -0,0 +1,195 @@
+<script lang="ts">
+	import Fuse from 'fuse.js';
+
+	import dayjs from 'dayjs';
+	import relativeTime from 'dayjs/plugin/relativeTime';
+	dayjs.extend(relativeTime);
+
+	import { toast } from 'svelte-sonner';
+	import { onMount, getContext } from 'svelte';
+	const i18n = getContext('i18n');
+
+	import { WEBUI_NAME, knowledge } from '$lib/stores';
+
+	import { getKnowledgeItems, deleteKnowledgeById } from '$lib/apis/knowledge';
+
+	import { blobToFile, transformFileName } from '$lib/utils';
+
+	import { goto } from '$app/navigation';
+	import Tooltip from '../common/Tooltip.svelte';
+	import GarbageBin from '../icons/GarbageBin.svelte';
+	import Pencil from '../icons/Pencil.svelte';
+	import DeleteConfirmDialog from '../common/ConfirmDialog.svelte';
+	import ItemMenu from './Knowledge/ItemMenu.svelte';
+
+	let query = '';
+	let selectedItem = null;
+	let showDeleteConfirm = false;
+
+	let fuse = null;
+
+	let filteredItems = [];
+	$: if (fuse) {
+		filteredItems = query
+			? fuse.search(query).map((e) => {
+					return e.item;
+				})
+			: $knowledge;
+	}
+
+	const deleteHandler = async (item) => {
+		const res = await deleteKnowledgeById(localStorage.token, item.id).catch((e) => {
+			toast.error(e);
+		});
+
+		if (res) {
+			knowledge.set(await getKnowledgeItems(localStorage.token));
+			toast.success($i18n.t('Knowledge deleted successfully.'));
+		}
+	};
+
+	onMount(async () => {
+		knowledge.set(await getKnowledgeItems(localStorage.token));
+
+		knowledge.subscribe((value) => {
+			fuse = new Fuse(value, {
+				keys: ['name', 'description']
+			});
+		});
+	});
+</script>
+
+<svelte:head>
+	<title>
+		{$i18n.t('Knowledge')} | {$WEBUI_NAME}
+	</title>
+</svelte:head>
+
+<DeleteConfirmDialog
+	bind:show={showDeleteConfirm}
+	on:confirm={() => {
+		deleteHandler(selectedItem);
+	}}
+/>
+
+<div class="mb-3">
+	<div class="flex justify-between items-center">
+		<div class="flex md:self-center text-lg font-medium px-0.5">
+			{$i18n.t('Knowledge')}
+			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-200 dark:bg-gray-700" />
+			<span class="text-lg font-medium text-gray-500 dark:text-gray-300">{$knowledge.length}</span>
+		</div>
+	</div>
+</div>
+
+<div class=" flex w-full space-x-2">
+	<div class="flex flex-1">
+		<div class=" self-center ml-1 mr-3">
+			<svg
+				xmlns="http://www.w3.org/2000/svg"
+				viewBox="0 0 20 20"
+				fill="currentColor"
+				class="w-4 h-4"
+			>
+				<path
+					fill-rule="evenodd"
+					d="M9 3.5a5.5 5.5 0 100 11 5.5 5.5 0 000-11zM2 9a7 7 0 1112.452 4.391l3.328 3.329a.75.75 0 11-1.06 1.06l-3.329-3.328A7 7 0 012 9z"
+					clip-rule="evenodd"
+				/>
+			</svg>
+		</div>
+		<input
+			class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-none bg-transparent"
+			bind:value={query}
+			placeholder={$i18n.t('Search Knowledge')}
+		/>
+	</div>
+
+	<div>
+		<button
+			class=" px-2 py-2 rounded-xl border border-gray-50 dark:border-gray-800 dark:border-0 hover:bg-gray-100 dark:bg-gray-800 dark:hover:bg-gray-700 transition font-medium text-sm flex items-center space-x-1"
+			aria-label={$i18n.t('Create Knowledge')}
+			on:click={() => {
+				goto('/workspace/knowledge/create');
+			}}
+		>
+			<svg
+				xmlns="http://www.w3.org/2000/svg"
+				viewBox="0 0 16 16"
+				fill="currentColor"
+				class="w-4 h-4"
+			>
+				<path
+					d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
+				/>
+			</svg>
+		</button>
+	</div>
+</div>
+
+<hr class=" border-gray-50 dark:border-gray-850 my-2.5" />
+
+<div class="my-3 mb-5 grid lg:grid-cols-2 xl:grid-cols-3 gap-2">
+	{#each filteredItems as item}
+		<button
+			class=" flex space-x-4 cursor-pointer text-left w-full px-4 py-3 border border-gray-50 dark:border-gray-850 hover:bg-gray-50 dark:hover:bg-gray-850 transition rounded-xl"
+			on:click={() => {
+				if (item?.meta?.document) {
+					toast.error(
+						$i18n.t(
+							'Only collections can be edited, create a new knowledge base to edit/add documents.'
+						)
+					);
+				} else {
+					goto(`/workspace/knowledge/${item.id}`);
+				}
+			}}
+		>
+			<div class=" w-full">
+				<div class="flex items-center justify-between -mt-1">
+					<div class=" font-semibold line-clamp-1 h-fit">{item.name}</div>
+
+					<div class=" flex self-center">
+						<ItemMenu
+							on:delete={() => {
+								selectedItem = item;
+								showDeleteConfirm = true;
+							}}
+						/>
+					</div>
+				</div>
+
+				<div class=" self-center flex-1">
+					<div class=" text-xs overflow-hidden text-ellipsis line-clamp-1">
+						{item.description}
+					</div>
+
+					<div class="mt-5 flex justify-between">
+						<div>
+							{#if item?.meta?.document}
+								<div
+									class="bg-gray-500/20 text-gray-700 dark:text-gray-200 rounded uppercase text-xs font-bold px-1"
+								>
+									{$i18n.t('Document')}
+								</div>
+							{:else}
+								<div
+									class="bg-green-500/20 text-green-700 dark:text-green-200 rounded uppercase text-xs font-bold px-1"
+								>
+									{$i18n.t('Collection')}
+								</div>
+							{/if}
+						</div>
+						<div class=" text-xs text-gray-500">
+							Updated {dayjs(item.updated_at * 1000).fromNow()}
+						</div>
+					</div>
+				</div>
+			</div>
+		</button>
+	{/each}
+</div>
+
+<div class=" text-gray-500 text-xs mt-1 mb-2">
+	ⓘ {$i18n.t("Use '#' in the prompt input to load and include your knowledge.")}
+</div>

src/lib/components/workspace/Knowledge/Collection.svelte

@@ -0,0 +1,479 @@
+<script lang="ts">
+	import { toast } from 'svelte-sonner';
+
+	import { onMount, getContext, onDestroy } from 'svelte';
+	const i18n = getContext('i18n');
+
+	import { goto } from '$app/navigation';
+	import { page } from '$app/stores';
+	import { mobile, showSidebar } from '$lib/stores';
+
+	import { updateFileDataContentById, uploadFile } from '$lib/apis/files';
+	import {
+		addFileToKnowledgeById,
+		getKnowledgeById,
+		removeFileFromKnowledgeById,
+		updateFileFromKnowledgeById,
+		updateKnowledgeById
+	} from '$lib/apis/knowledge';
+
+	import Spinner from '$lib/components/common/Spinner.svelte';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
+	import Badge from '$lib/components/common/Badge.svelte';
+	import Files from './Collection/Files.svelte';
+	import AddFilesPlaceholder from '$lib/components/AddFilesPlaceholder.svelte';
+	import AddContentModal from './Collection/AddTextContentModal.svelte';
+	import { transcribeAudio } from '$lib/apis/audio';
+	import { blobToFile } from '$lib/utils';
+	import { processFile } from '$lib/apis/retrieval';
+	import AddContentMenu from './Collection/AddContentMenu.svelte';
+	import AddTextContentModal from './Collection/AddTextContentModal.svelte';
+	import Check from '$lib/components/icons/Check.svelte';
+	import FloppyDisk from '$lib/components/icons/FloppyDisk.svelte';
+
+	let largeScreen = true;
+
+	type Knowledge = {
+		id: string;
+		name: string;
+		description: string;
+		data: {
+			file_ids: string[];
+		};
+		files: any[];
+	};
+
+	let id = null;
+	let knowledge: Knowledge | null = null;
+	let query = '';
+
+	let showAddTextContentModal = false;
+	let inputFiles = null;
+
+	let selectedFile = null;
+	let selectedFileId = null;
+
+	$: if (selectedFileId) {
+		const file = knowledge.files.find((file) => file.id === selectedFileId);
+		if (file) {
+			file.data = file.data ?? { content: '' };
+			selectedFile = file;
+		}
+	} else {
+		selectedFile = null;
+	}
+
+	let debounceTimeout = null;
+	let mediaQuery;
+	let dragged = false;
+
+	const createFileFromText = (name, content) => {
+		const blob = new Blob([content], { type: 'text/plain' });
+		const file = blobToFile(blob, `${name}.md`);
+
+		console.log(file);
+		return file;
+	};
+
+	const uploadFileHandler = async (file) => {
+		console.log(file);
+
+		// Check if the file is an audio file and transcribe/convert it to text file
+		if (['audio/mpeg', 'audio/wav', 'audio/ogg', 'audio/x-m4a'].includes(file['type'])) {
+			const res = await transcribeAudio(localStorage.token, file).catch((error) => {
+				toast.error(error);
+				return null;
+			});
+
+			if (res) {
+				console.log(res);
+				const blob = new Blob([res.text], { type: 'text/plain' });
+				file = blobToFile(blob, `${file.name}.txt`);
+			}
+		}
+
+		try {
+			const uploadedFile = await uploadFile(localStorage.token, file).catch((e) => {
+				toast.error(e);
+			});
+
+			if (uploadedFile) {
+				console.log(uploadedFile);
+				addFileHandler(uploadedFile.id);
+			} else {
+				toast.error($i18n.t('Failed to upload file.'));
+			}
+		} catch (e) {
+			toast.error(e);
+		}
+	};
+
+	const addFileHandler = async (fileId) => {
+		const updatedKnowledge = await addFileToKnowledgeById(localStorage.token, id, fileId).catch(
+			(e) => {
+				toast.error(e);
+			}
+		);
+
+		if (updatedKnowledge) {
+			knowledge = updatedKnowledge;
+			toast.success($i18n.t('File added successfully.'));
+		}
+	};
+
+	const deleteFileHandler = async (fileId) => {
+		const updatedKnowledge = await removeFileFromKnowledgeById(
+			localStorage.token,
+			id,
+			fileId
+		).catch((e) => {
+			toast.error(e);
+		});
+
+		if (updatedKnowledge) {
+			knowledge = updatedKnowledge;
+			toast.success($i18n.t('File removed successfully.'));
+		}
+	};
+
+	const updateFileContentHandler = async () => {
+		const fileId = selectedFile.id;
+		const content = selectedFile.data.content;
+
+		const res = updateFileDataContentById(localStorage.token, fileId, content).catch((e) => {
+			toast.error(e);
+		});
+
+		const updatedKnowledge = await updateFileFromKnowledgeById(
+			localStorage.token,
+			id,
+			fileId
+		).catch((e) => {
+			toast.error(e);
+		});
+
+		if (res && updatedKnowledge) {
+			knowledge = updatedKnowledge;
+			toast.success($i18n.t('File content updated successfully.'));
+		}
+	};
+
+	const changeDebounceHandler = () => {
+		console.log('debounce');
+		if (debounceTimeout) {
+			clearTimeout(debounceTimeout);
+		}
+
+		debounceTimeout = setTimeout(async () => {
+			const res = await updateKnowledgeById(localStorage.token, id, {
+				name: knowledge.name,
+				description: knowledge.description
+			}).catch((e) => {
+				toast.error(e);
+			});
+
+			if (res) {
+				toast.success($i18n.t('Knowledge updated successfully'));
+			}
+		}, 1000);
+	};
+
+	const handleMediaQuery = async (e) => {
+		if (e.matches) {
+			largeScreen = true;
+		} else {
+			largeScreen = false;
+		}
+	};
+
+	const onDragOver = (e) => {
+		e.preventDefault();
+		dragged = true;
+	};
+
+	const onDragLeave = () => {
+		dragged = false;
+	};
+
+	const onDrop = async (e) => {
+		e.preventDefault();
+
+		if (e.dataTransfer?.files) {
+			const inputFiles = e.dataTransfer?.files;
+
+			if (inputFiles && inputFiles.length > 0) {
+				for (const file of inputFiles) {
+					await uploadFileHandler(file);
+				}
+			} else {
+				toast.error($i18n.t(`File not found.`));
+			}
+		}
+
+		dragged = false;
+	};
+
+	onMount(async () => {
+		// listen to resize 1024px
+		mediaQuery = window.matchMedia('(min-width: 1024px)');
+
+		mediaQuery.addEventListener('change', handleMediaQuery);
+		handleMediaQuery(mediaQuery);
+
+		id = $page.params.id;
+
+		const res = await getKnowledgeById(localStorage.token, id).catch((e) => {
+			toast.error(e);
+			return null;
+		});
+
+		if (res) {
+			knowledge = res;
+		} else {
+			goto('/workspace/knowledge');
+		}
+
+		const dropZone = document.querySelector('body');
+		dropZone?.addEventListener('dragover', onDragOver);
+		dropZone?.addEventListener('drop', onDrop);
+		dropZone?.addEventListener('dragleave', onDragLeave);
+	});
+
+	onDestroy(() => {
+		mediaQuery?.removeEventListener('change', handleMediaQuery);
+		const dropZone = document.querySelector('body');
+		dropZone?.removeEventListener('dragover', onDragOver);
+		dropZone?.removeEventListener('drop', onDrop);
+		dropZone?.removeEventListener('dragleave', onDragLeave);
+	});
+</script>
+
+{#if dragged}
+	<div
+		class="fixed {$showSidebar
+			? 'left-0 md:left-[260px] md:w-[calc(100%-260px)]'
+			: 'left-0'}  w-full h-full flex z-50 touch-none pointer-events-none"
+		id="dropzone"
+		role="region"
+		aria-label="Drag and Drop Container"
+	>
+		<div class="absolute w-full h-full backdrop-blur bg-gray-800/40 flex justify-center">
+			<div class="m-auto pt-64 flex flex-col justify-center">
+				<div class="max-w-md">
+					<AddFilesPlaceholder>
+						<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
+							Drop any files here to add to my documents
+						</div>
+					</AddFilesPlaceholder>
+				</div>
+			</div>
+		</div>
+	</div>
+{/if}
+
+<AddTextContentModal
+	bind:show={showAddTextContentModal}
+	on:submit={(e) => {
+		const file = createFileFromText(e.detail.name, e.detail.content);
+		uploadFileHandler(file);
+	}}
+/>
+
+<input
+	id="files-input"
+	bind:files={inputFiles}
+	type="file"
+	multiple
+	hidden
+	on:change={() => {
+		if (inputFiles && inputFiles.length > 0) {
+			for (const file of inputFiles) {
+				uploadFileHandler(file);
+			}
+
+			inputFiles = null;
+			const fileInputElement = document.getElementById('files-input');
+
+			if (fileInputElement) {
+				fileInputElement.value = '';
+			}
+		} else {
+			toast.error($i18n.t(`File not found.`));
+		}
+	}}
+/>
+
+<div class="flex flex-col w-full max-h-[100dvh] h-full">
+	<button
+		class="flex space-x-1"
+		on:click={() => {
+			goto('/workspace/knowledge');
+		}}
+	>
+		<div class=" self-center">
+			<svg
+				xmlns="http://www.w3.org/2000/svg"
+				viewBox="0 0 20 20"
+				fill="currentColor"
+				class="w-4 h-4"
+			>
+				<path
+					fill-rule="evenodd"
+					d="M17 10a.75.75 0 01-.75.75H5.612l4.158 3.96a.75.75 0 11-1.04 1.08l-5.5-5.25a.75.75 0 010-1.08l5.5-5.25a.75.75 0 111.04 1.08L5.612 9.25H16.25A.75.75 0 0117 10z"
+					clip-rule="evenodd"
+				/>
+			</svg>
+		</div>
+		<div class=" self-center font-medium text-sm">{$i18n.t('Back')}</div>
+	</button>
+
+	<div class="flex flex-col my-2 flex-1 overflow-auto h-0">
+		{#if id && knowledge}
+			<div class=" flex w-full mt-1 mb-3.5">
+				<div class="flex-1">
+					<div class="flex items-center justify-between w-full px-0.5 mb-1">
+						<div class="w-full">
+							<input
+								type="text"
+								class="w-full font-medium text-2xl font-primary bg-transparent outline-none"
+								bind:value={knowledge.name}
+								on:input={() => {
+									changeDebounceHandler();
+								}}
+							/>
+						</div>
+
+						<div class=" flex-shrink-0">
+							<div>
+								<Badge type="success" content="Collection" />
+							</div>
+						</div>
+					</div>
+
+					<div class="flex w-full px-1">
+						<input
+							type="text"
+							class="w-full text-gray-500 text-sm bg-transparent outline-none"
+							bind:value={knowledge.description}
+							on:input={() => {
+								changeDebounceHandler();
+							}}
+						/>
+					</div>
+				</div>
+			</div>
+
+			<div class="flex flex-row h-0 flex-1 overflow-auto">
+				<div
+					class=" {largeScreen
+						? 'flex-shrink-0'
+						: 'flex-1'} flex py-2.5 w-80 rounded-2xl border border-gray-50 dark:border-gray-850"
+				>
+					<div class=" flex flex-col w-full space-x-2 rounded-lg h-full">
+						<div class="w-full h-full flex flex-col">
+							<div class=" px-3">
+								<div class="flex">
+									<div class=" self-center ml-1 mr-3">
+										<svg
+											xmlns="http://www.w3.org/2000/svg"
+											viewBox="0 0 20 20"
+											fill="currentColor"
+											class="w-4 h-4"
+										>
+											<path
+												fill-rule="evenodd"
+												d="M9 3.5a5.5 5.5 0 100 11 5.5 5.5 0 000-11zM2 9a7 7 0 1112.452 4.391l3.328 3.329a.75.75 0 11-1.06 1.06l-3.329-3.328A7 7 0 012 9z"
+												clip-rule="evenodd"
+											/>
+										</svg>
+									</div>
+									<input
+										class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-none bg-transparent"
+										bind:value={query}
+										placeholder={$i18n.t('Search Collection')}
+									/>
+
+									<div>
+										<AddContentMenu
+											on:files={() => {
+												document.getElementById('files-input').click();
+											}}
+											on:text={() => {
+												showAddTextContentModal = true;
+											}}
+										/>
+									</div>
+								</div>
+
+								<hr class=" mt-2 mb-1 border-gray-50 dark:border-gray-850" />
+							</div>
+
+							{#if (knowledge?.files ?? []).length > 0}
+								<div class=" flex overflow-y-auto h-full w-full scrollbar-hidden text-xs">
+									<Files
+										files={knowledge.files}
+										{selectedFileId}
+										on:click={(e) => {
+											selectedFileId = e.detail;
+										}}
+										on:delete={(e) => {
+											console.log(e.detail);
+
+											selectedFileId = null;
+											deleteFileHandler(e.detail);
+										}}
+									/>
+								</div>
+							{:else}
+								<div class="m-auto text-gray-500 text-xs">No content found</div>
+							{/if}
+						</div>
+					</div>
+				</div>
+
+				{#if largeScreen}
+					<div class="flex-1 flex justify-start max-h-full overflow-hidden pl-3">
+						{#if selectedFile}
+							<div class=" flex flex-col w-full h-full">
+								<div class=" flex-shrink-0 mb-2 flex items-center">
+									<div class=" flex-1 text-xl line-clamp-1">
+										{selectedFile?.meta?.name}
+									</div>
+
+									<div>
+										<button
+											class="self-center w-fit text-sm py-1 px-2.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-lg"
+											on:click={() => {
+												updateFileContentHandler();
+											}}
+										>
+											{$i18n.t('Save')}
+										</button>
+									</div>
+								</div>
+
+								<div class=" flex-grow">
+									<textarea
+										class=" w-full h-full resize-none rounded-xl py-4 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
+										bind:value={selectedFile.data.content}
+										placeholder={$i18n.t('Add content here')}
+									/>
+								</div>
+							</div>
+						{:else}
+							<div class="m-auto">
+								<AddFilesPlaceholder title={$i18n.t('Select/Add Files')}>
+									<div class=" mt-2 text-center text-sm dark:text-gray-200 w-full">
+										Select a file to view or drag and drop a file to upload
+									</div>
+								</AddFilesPlaceholder>
+							</div>
+						{/if}
+					</div>
+				{/if}
+			</div>
+		{:else}
+			<Spinner />
+		{/if}
+	</div>
+</div>

src/lib/components/workspace/Knowledge/Collection/AddContentMenu.svelte

@@ -0,0 +1,86 @@
+<script lang="ts">
+	import { DropdownMenu } from 'bits-ui';
+	import { flyAndScale } from '$lib/utils/transitions';
+	import { getContext, createEventDispatcher } from 'svelte';
+	const dispatch = createEventDispatcher();
+
+	import Dropdown from '$lib/components/common/Dropdown.svelte';
+	import GarbageBin from '$lib/components/icons/GarbageBin.svelte';
+	import Pencil from '$lib/components/icons/Pencil.svelte';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
+	import Tags from '$lib/components/chat/Tags.svelte';
+	import Share from '$lib/components/icons/Share.svelte';
+	import ArchiveBox from '$lib/components/icons/ArchiveBox.svelte';
+	import DocumentDuplicate from '$lib/components/icons/DocumentDuplicate.svelte';
+	import ArrowDownTray from '$lib/components/icons/ArrowDownTray.svelte';
+	import ArrowUpCircle from '$lib/components/icons/ArrowUpCircle.svelte';
+	import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte';
+	import BarsArrowUp from '$lib/components/icons/BarsArrowUp.svelte';
+
+	const i18n = getContext('i18n');
+
+	export let onClose: Function = () => {};
+
+	let show = false;
+</script>
+
+<Dropdown
+	bind:show
+	on:change={(e) => {
+		if (e.detail === false) {
+			onClose();
+		}
+	}}
+	align="end"
+>
+	<Tooltip content={$i18n.t('Add Content')}>
+		<button
+			class=" px-2 py-2 rounded-xl border border-gray-50 dark:border-gray-600 dark:border-0 hover:bg-gray-100 dark:bg-gray-800 dark:hover:bg-gray-700 transition font-medium text-sm flex items-center space-x-1"
+			on:click={(e) => {
+				e.stopPropagation();
+				show = true;
+			}}
+		>
+			<svg
+				xmlns="http://www.w3.org/2000/svg"
+				viewBox="0 0 16 16"
+				fill="currentColor"
+				class="w-4 h-4"
+			>
+				<path
+					d="M8.75 3.75a.75.75 0 0 0-1.5 0v3.5h-3.5a.75.75 0 0 0 0 1.5h3.5v3.5a.75.75 0 0 0 1.5 0v-3.5h3.5a.75.75 0 0 0 0-1.5h-3.5v-3.5Z"
+				/>
+			</svg>
+		</button>
+	</Tooltip>
+
+	<div slot="content">
+		<DropdownMenu.Content
+			class="w-full max-w-44 rounded-xl p-1 z-50 bg-white dark:bg-gray-850 dark:text-white shadow"
+			sideOffset={4}
+			side="bottom"
+			align="end"
+			transition={flyAndScale}
+		>
+			<DropdownMenu.Item
+				class="flex  gap-2  items-center px-3 py-2 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
+				on:click={() => {
+					dispatch('files');
+				}}
+			>
+				<ArrowUpCircle strokeWidth="2" />
+				<div class="flex items-center">{$i18n.t('Upload files')}</div>
+			</DropdownMenu.Item>
+
+			<DropdownMenu.Item
+				class="flex  gap-2  items-center px-3 py-2 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
+				on:click={() => {
+					dispatch('text');
+				}}
+			>
+				<BarsArrowUp strokeWidth="2" />
+				<div class="flex items-center">{$i18n.t('Add text content')}</div>
+			</DropdownMenu.Item>
+		</DropdownMenu.Content>
+	</div>
+</Dropdown>

src/lib/components/workspace/Knowledge/Collection/AddTextContentModal.svelte

@@ -0,0 +1,118 @@
+<script lang="ts">
+	import { toast } from 'svelte-sonner';
+	import dayjs from 'dayjs';
+
+	import { onMount, getContext, createEventDispatcher } from 'svelte';
+	const i18n = getContext('i18n');
+	const dispatch = createEventDispatcher();
+
+	import Modal from '$lib/components/common/Modal.svelte';
+	export let show = false;
+
+	let name = '';
+	let content = '';
+</script>
+
+<Modal size="md" bind:show>
+	<div>
+		<div class=" flex justify-between dark:text-gray-300 px-5 pt-4">
+			<div class=" text-lg font-medium self-center">{$i18n.t('Add Content')}</div>
+			<button
+				class="self-center"
+				on:click={() => {
+					show = false;
+				}}
+			>
+				<svg
+					xmlns="http://www.w3.org/2000/svg"
+					viewBox="0 0 20 20"
+					fill="currentColor"
+					class="w-5 h-5"
+				>
+					<path
+						d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
+					/>
+				</svg>
+			</button>
+		</div>
+		<div class="flex flex-col md:flex-row w-full px-5 py-4 md:space-x-4 dark:text-gray-200">
+			<div class=" flex flex-col w-full sm:flex-row sm:justify-center sm:space-x-6">
+				<form
+					class="flex flex-col w-full"
+					on:submit|preventDefault={() => {
+						dispatch('submit', {
+							name,
+							content
+						});
+						show = false;
+						name = '';
+						content = '';
+					}}
+				>
+					<div class="mb-3 w-full">
+						<div class="w-full flex flex-col gap-2.5">
+							<div class="w-full">
+								<div class=" text-sm mb-2">Title</div>
+
+								<div class="w-full mt-1">
+									<input
+										class="w-full rounded-lg py-2 px-4 text-sm bg-white dark:text-gray-300 dark:bg-gray-850 outline-none"
+										type="text"
+										bind:value={name}
+										placeholder={`Name your content`}
+										required
+									/>
+								</div>
+							</div>
+
+							<div>
+								<div class="text-sm mb-2">Content</div>
+
+								<div class=" w-full mt-1">
+									<textarea
+										class="w-full resize-none rounded-lg py-2 px-4 text-sm bg-whites dark:text-gray-300 dark:bg-gray-850 outline-none"
+										rows="10"
+										bind:value={content}
+										placeholder={`Write your content here`}
+										required
+									/>
+								</div>
+							</div>
+						</div>
+					</div>
+
+					<div class="flex justify-end text-sm font-medium">
+						<button
+							class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
+							type="submit"
+						>
+							{$i18n.t('Add Content')}
+						</button>
+					</div>
+				</form>
+			</div>
+		</div>
+	</div>
+</Modal>
+
+<style>
+	input::-webkit-outer-spin-button,
+	input::-webkit-inner-spin-button {
+		/* display: none; <- Crashes Chrome on hover */
+		-webkit-appearance: none;
+		margin: 0; /* <-- Apparently some margin are still there even though it's hidden */
+	}
+
+	.tabs::-webkit-scrollbar {
+		display: none; /* for Chrome, Safari and Opera */
+	}
+
+	.tabs {
+		-ms-overflow-style: none; /* IE and Edge */
+		scrollbar-width: none; /* Firefox */
+	}
+
+	input[type='number'] {
+		-moz-appearance: textfield; /* Firefox */
+	}
+</style>

src/lib/components/workspace/Knowledge/Collection/Files.svelte

@@ -0,0 +1,33 @@
+<script lang="ts">
+	import { createEventDispatcher } from 'svelte';
+	const dispatch = createEventDispatcher();
+
+	import FileItem from '$lib/components/common/FileItem.svelte';
+
+	export let selectedFileId = null;
+	export let files = [];
+</script>
+
+<div class=" max-h-full flex flex-col w-full">
+	{#each files as file (file.id)}
+		<div class="mt-2 px-2">
+			<FileItem
+				className="w-full"
+				colorClassName="{selectedFileId === file.id
+					? ' bg-gray-50 dark:bg-gray-850'
+					: 'bg-transparent'} hover:bg-gray-50 dark:hover:bg-gray-850 transition"
+				{file}
+				name={file.meta.name}
+				type="file"
+				size={file.meta?.size ?? ''}
+				dismissible
+				on:click={() => {
+					dispatch('click', file.id);
+				}}
+				on:dismiss={() => {
+					dispatch('delete', file.id);
+				}}
+			/>
+		</div>
+	{/each}
+</div>

src/lib/components/workspace/Knowledge/CreateCollection.svelte

@@ -0,0 +1,138 @@
+<script>
+	import { goto } from '$app/navigation';
+	import { getContext } from 'svelte';
+	const i18n = getContext('i18n');
+
+	import { createNewKnowledge, getKnowledgeItems } from '$lib/apis/knowledge';
+	import { toast } from 'svelte-sonner';
+	import { knowledge } from '$lib/stores';
+
+	let loading = false;
+
+	let name = '';
+	let description = '';
+
+	const submitHandler = async () => {
+		loading = true;
+
+		const res = await createNewKnowledge(localStorage.token, name, description).catch((e) => {
+			toast.error(e);
+		});
+
+		if (res) {
+			toast.success($i18n.t('Knowledge created successfully.'));
+			knowledge.set(await getKnowledgeItems(localStorage.token));
+			goto(`/workspace/knowledge/${res.id}`);
+		}
+
+		loading = false;
+	};
+</script>
+
+<div class="w-full max-h-full">
+	<button
+		class="flex space-x-1"
+		on:click={() => {
+			goto('/workspace/knowledge');
+		}}
+	>
+		<div class=" self-center">
+			<svg
+				xmlns="http://www.w3.org/2000/svg"
+				viewBox="0 0 20 20"
+				fill="currentColor"
+				class="w-4 h-4"
+			>
+				<path
+					fill-rule="evenodd"
+					d="M17 10a.75.75 0 01-.75.75H5.612l4.158 3.96a.75.75 0 11-1.04 1.08l-5.5-5.25a.75.75 0 010-1.08l5.5-5.25a.75.75 0 111.04 1.08L5.612 9.25H16.25A.75.75 0 0117 10z"
+					clip-rule="evenodd"
+				/>
+			</svg>
+		</div>
+		<div class=" self-center font-medium text-sm">{$i18n.t('Back')}</div>
+	</button>
+
+	<form
+		class="flex flex-col max-w-lg mx-auto mt-10 mb-10"
+		on:submit|preventDefault={() => {
+			submitHandler();
+		}}
+	>
+		<div class=" w-full flex flex-col justify-center">
+			<div class=" text-2xl font-medium font-primary mb-2.5">Create a knowledge base</div>
+
+			<div class="w-full flex flex-col gap-2.5">
+				<div class="w-full">
+					<div class=" text-sm mb-2">What are you working on?</div>
+
+					<div class="w-full mt-1">
+						<input
+							class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
+							type="text"
+							bind:value={name}
+							placeholder={`Name your knowledge base`}
+							required
+						/>
+					</div>
+				</div>
+
+				<div>
+					<div class="text-sm mb-2">What are you trying to achieve?</div>
+
+					<div class=" w-full mt-1">
+						<textarea
+							class="w-full resize-none rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
+							rows="4"
+							bind:value={description}
+							placeholder={`Describe your knowledge base and objectives`}
+							required
+						/>
+					</div>
+				</div>
+			</div>
+		</div>
+
+		<div class="flex justify-end mt-2">
+			<div>
+				<button
+					class=" text-sm px-4 py-2 transition rounded-lg {loading
+						? ' cursor-not-allowed bg-gray-100 dark:bg-gray-800'
+						: ' bg-gray-50 hover:bg-gray-100 dark:bg-gray-850 dark:hover:bg-gray-800'} flex"
+					type="submit"
+					disabled={loading}
+				>
+					<div class=" self-center font-medium">{$i18n.t('Create Knowledge')}</div>
+
+					{#if loading}
+						<div class="ml-1.5 self-center">
+							<svg
+								class=" w-4 h-4"
+								viewBox="0 0 24 24"
+								fill="currentColor"
+								xmlns="http://www.w3.org/2000/svg"
+								><style>
+									.spinner_ajPY {
+										transform-origin: center;
+										animation: spinner_AtaB 0.75s infinite linear;
+									}
+									@keyframes spinner_AtaB {
+										100% {
+											transform: rotate(360deg);
+										}
+									}
+								</style><path
+									d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,19a8,8,0,1,1,8-8A8,8,0,0,1,12,20Z"
+									opacity=".25"
+								/><path
+									d="M10.14,1.16a11,11,0,0,0-9,8.92A1.59,1.59,0,0,0,2.46,12,1.52,1.52,0,0,0,4.11,10.7a8,8,0,0,1,6.66-6.61A1.42,1.42,0,0,0,12,2.69h0A1.57,1.57,0,0,0,10.14,1.16Z"
+									class="spinner_ajPY"
+								/></svg
+							>
+						</div>
+					{/if}
+				</button>
+			</div>
+		</div>
+	</form>
+</div>

src/lib/components/workspace/Knowledge/ItemMenu.svelte

@@ -0,0 +1,69 @@
+<script lang="ts">
+	import { DropdownMenu } from 'bits-ui';
+	import { flyAndScale } from '$lib/utils/transitions';
+	import { getContext, createEventDispatcher } from 'svelte';
+	const dispatch = createEventDispatcher();
+
+	import Dropdown from '$lib/components/common/Dropdown.svelte';
+	import GarbageBin from '$lib/components/icons/GarbageBin.svelte';
+	import Pencil from '$lib/components/icons/Pencil.svelte';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
+	import Tags from '$lib/components/chat/Tags.svelte';
+	import Share from '$lib/components/icons/Share.svelte';
+	import ArchiveBox from '$lib/components/icons/ArchiveBox.svelte';
+	import DocumentDuplicate from '$lib/components/icons/DocumentDuplicate.svelte';
+	import ArrowDownTray from '$lib/components/icons/ArrowDownTray.svelte';
+	import ArrowUpCircle from '$lib/components/icons/ArrowUpCircle.svelte';
+	import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte';
+
+	const i18n = getContext('i18n');
+
+	export let onClose: Function = () => {};
+
+	let show = false;
+</script>
+
+<Dropdown
+	bind:show
+	on:change={(e) => {
+		if (e.detail === false) {
+			onClose();
+		}
+	}}
+	align="end"
+>
+	<Tooltip content={$i18n.t('More')}>
+		<slot
+			><button
+				class="self-center w-fit text-sm p-1.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-xl"
+				type="button"
+				on:click={(e) => {
+					e.stopPropagation();
+					show = true;
+				}}
+			>
+				<EllipsisHorizontal className="size-5" />
+			</button>
+		</slot>
+	</Tooltip>
+
+	<div slot="content">
+		<DropdownMenu.Content
+			class="w-full max-w-[160px] rounded-xl px-1 py-1.5 border border-gray-300/30 dark:border-gray-700/50 z-50 bg-white dark:bg-gray-850 dark:text-white shadow"
+			sideOffset={-2}
+			side="bottom"
+			align="end"
+			transition={flyAndScale}
+		>
+			<DropdownMenu.Item
+				class="flex  gap-2  items-center px-3 py-2 text-sm  font-medium cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
+				on:click={() => {
+					dispatch('delete');
+				}}
+			>
+				<GarbageBin strokeWidth="2" />
+				<div class="flex items-center">{$i18n.t('Delete')}</div>
+			</DropdownMenu.Item>
+		</DropdownMenu.Content>
+	</div>
+</Dropdown>

src/lib/components/workspace/Models/Knowledge.svelte

@@ -1,9 +1,9 @@
 <script lang="ts">
 	import { getContext } from 'svelte';
 	import Selector from './Knowledge/Selector.svelte';
+	import FileItem from '$lib/components/common/FileItem.svelte';
 
 	export let knowledge = [];
-
 	const i18n = getContext('i18n');
 </script>
 
@@ -13,91 +13,44 @@
 	</div>
 
 	<div class=" text-xs dark:text-gray-500">
-		{$i18n.t('To add documents here, upload them to the "Documents" workspace first.')}
+		{$i18n.t('To attach knowledge base here, add them to the "Knowledge" workspace first.')}
 	</div>
 
 	<div class="flex flex-col">
-		{#if knowledge.length > 0}
+		{#if knowledge?.length > 0}
 			<div class=" flex items-center gap-2 mt-2">
 				{#each knowledge as file, fileIdx}
-					<div class=" relative group">
+					<FileItem
-						<div
+						{file}
-							class="h-16 w-[15rem] flex items-center space-x-3 px-2.5 dark:bg-gray-600 rounded-xl border border-gray-200 dark:border-none"
+						dismissible
-						>
+						on:dismiss={(e) => {
-							<div class="p-2.5 bg-red-400 text-white rounded-lg">
+							knowledge = knowledge.filter((_, idx) => idx !== fileIdx);
-								{#if (file?.type ?? 'doc') === 'doc'}
+						}}
-									<svg
+					/>
-										xmlns="http://www.w3.org/2000/svg"
-										viewBox="0 0 24 24"
-										fill="currentColor"
-										class="size-6"
-									>
-										<path
-											fill-rule="evenodd"
-											d="M5.625 1.5c-1.036 0-1.875.84-1.875 1.875v17.25c0 1.035.84 1.875 1.875 1.875h12.75c1.035 0 1.875-.84 1.875-1.875V12.75A3.75 3.75 0 0 0 16.5 9h-1.875a1.875 1.875 0 0 1-1.875-1.875V5.25A3.75 3.75 0 0 0 9 1.5H5.625ZM7.5 15a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 7.5 15Zm.75 2.25a.75.75 0 0 0 0 1.5H12a.75.75 0 0 0 0-1.5H8.25Z"
-											clip-rule="evenodd"
-										/>
-										<path
-											d="M12.971 1.816A5.23 5.23 0 0 1 14.25 5.25v1.875c0 .207.168.375.375.375H16.5a5.23 5.23 0 0 1 3.434 1.279 9.768 9.768 0 0 0-6.963-6.963Z"
-										/>
-									</svg>
-								{:else if file.type === 'collection'}
-									<svg
-										xmlns="http://www.w3.org/2000/svg"
-										viewBox="0 0 24 24"
-										fill="currentColor"
-										class="size-6"
-									>
-										<path
-											d="M7.5 3.375c0-1.036.84-1.875 1.875-1.875h.375a3.75 3.75 0 0 1 3.75 3.75v1.875C13.5 8.161 14.34 9 15.375 9h1.875A3.75 3.75 0 0 1 21 12.75v3.375C21 17.16 20.16 18 19.125 18h-9.75A1.875 1.875 0 0 1 7.5 16.125V3.375Z"
-										/>
-										<path
-											d="M15 5.25a5.23 5.23 0 0 0-1.279-3.434 9.768 9.768 0 0 1 6.963 6.963A5.23 5.23 0 0 0 17.25 7.5h-1.875A.375.375 0 0 1 15 7.125V5.25ZM4.875 6H6v10.125A3.375 3.375 0 0 0 9.375 19.5H16.5v1.125c0 1.035-.84 1.875-1.875 1.875h-9.75A1.875 1.875 0 0 1 3 20.625V7.875C3 6.839 3.84 6 4.875 6Z"
-										/>
-									</svg>
-								{/if}
-							</div>
-
-							<div class="flex flex-col justify-center -space-y-0.5">
-								<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1">
-									{file?.title ?? `#${file.name}`}
-								</div>
-
-								<div class=" text-gray-500 text-sm">{$i18n.t(file?.type ?? 'Document')}</div>
-							</div>
-						</div>
-
-						<div class=" absolute -top-1 -right-1">
-							<button
-								class=" bg-gray-400 text-white border border-white rounded-full group-hover:visible invisible transition"
-								type="button"
-								on:click={() => {
-									knowledge.splice(fileIdx, 1);
-									knowledge = knowledge;
-								}}
-							>
-								<svg
-									xmlns="http://www.w3.org/2000/svg"
-									viewBox="0 0 20 20"
-									fill="currentColor"
-									class="w-4 h-4"
-								>
-									<path
-										d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
-									/>
-								</svg>
-							</button>
-						</div>
-					</div>
 				{/each}
 			</div>
 		{/if}
 
 		<div class="flex flex-wrap text-sm font-medium gap-1.5 mt-2">
-			<Selector bind:knowledge>
+			<Selector
+				bind:knowledge
+				on:select={(e) => {
+					const item = e.detail;
+
+					if (!knowledge.find((k) => k.name === item.name)) {
+						knowledge = [
+							...knowledge,
+							{
+								...item,
+								type: item?.type ?? 'doc'
+							}
+						];
+					}
+				}}
+			>
 				<button
 					class=" px-3.5 py-1.5 font-medium hover:bg-black/5 dark:hover:bg-white/5 outline outline-1 outline-gray-300 dark:outline-gray-800 rounded-3xl"
-					type="button">{$i18n.t('Select Documents')}</button
+					type="button">{$i18n.t('Select Knowledge')}</button
 				>
 			</Selector>
 		</div>

src/lib/components/workspace/Models/Knowledge/Selector.svelte

@@ -1,46 +1,50 @@
 <script lang="ts">
 	import { DropdownMenu } from 'bits-ui';
-
+	import { onMount, getContext, createEventDispatcher } from 'svelte';
-	import { documents } from '$lib/stores';
 	import { flyAndScale } from '$lib/utils/transitions';
-
+	import { knowledge } from '$lib/stores';
 	import Dropdown from '$lib/components/common/Dropdown.svelte';
-	import { onMount, getContext } from 'svelte';
 
 	const i18n = getContext('i18n');
+	const dispatch = createEventDispatcher();
 
 	export let onClose: Function = () => {};
 
-	export let knowledge = [];
-
 	let items = [];
 
 	onMount(() => {
-		let collections = [
+		let legacy_documents = $knowledge.filter((item) => item?.meta?.document);
-			...($documents.length > 0
+		let legacy_collections =
+			legacy_documents.length > 0
 				? [
 						{
 							name: 'All Documents',
+							legacy: true,
 							type: 'collection',
-							title: $i18n.t('All Documents'),
+							description: 'Deprecated (legacy collection), please create a new knowledge base.',
-							collection_names: $documents.map((doc) => doc.collection_name)
-						}
-					]
-				: []),
-			...$documents
-				.reduce((a, e, i, arr) => {
-					return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])];
-				}, [])
-				.map((tag) => ({
-					name: tag,
-					type: 'collection',
-					collection_names: $documents
-						.filter((doc) => (doc?.content?.tags ?? []).map((tag) => tag.name).includes(tag))
-						.map((doc) => doc.collection_name)
-				}))
-		];
 
-		items = [...collections, ...$documents];
+							title: $i18n.t('All Documents'),
+							collection_names: legacy_documents.map((item) => item.id)
+						},
+
+						...legacy_documents
+							.reduce((a, item) => {
+								return [...new Set([...a, ...(item?.meta?.tags ?? []).map((tag) => tag.name)])];
+							}, [])
+							.map((tag) => ({
+								name: tag,
+								legacy: true,
+								type: 'collection',
+								description: 'Deprecated (legacy collection), please create a new knowledge base.',
+
+								collection_names: legacy_documents
+									.filter((item) => (item?.meta?.tags ?? []).map((tag) => tag.name).includes(tag))
+									.map((item) => item.id)
+							}))
+					]
+				: [];
+
+		items = [...$knowledge, ...legacy_collections];
 	});
 </script>
 
@@ -55,7 +59,7 @@
 
 	<div slot="content">
 		<DropdownMenu.Content
-			class="w-full max-w-[300px]  rounded-lg px-1 py-1.5 border border-gray-300/30 dark:border-gray-700/50 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
+			class="w-full max-w-80 rounded-lg px-1 py-1.5 border border-gray-300/30 dark:border-gray-700/50 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
 			sideOffset={8}
 			side="bottom"
 			align="start"
@@ -64,64 +68,38 @@
 			<div class="max-h-[10rem] overflow-y-scroll">
 				{#if items.length === 0}
 					<div class="text-center text-sm text-gray-500 dark:text-gray-400">
-						{$i18n.t('No documents found')}
+						{$i18n.t('No knowledge found')}
 					</div>
 				{:else}
 					{#each items as item}
 						<DropdownMenu.Item
 							class="flex gap-2.5 items-center px-3 py-2 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
 							on:click={() => {
-								if (!knowledge.find((k) => k.name === item.name)) {
+								dispatch('select', item);
-									knowledge = [
-										...knowledge,
-										{
-											...item,
-											type: item?.type ?? 'doc'
-										}
-									];
-								}
 							}}
 						>
-							<div class="flex self-start">
-								{#if (item?.type ?? 'doc') === 'doc'}
-									<svg
-										xmlns="http://www.w3.org/2000/svg"
-										viewBox="0 0 24 24"
-										fill="currentColor"
-										class="w-4"
-									>
-										<path
-											fill-rule="evenodd"
-											d="M5.625 1.5c-1.036 0-1.875.84-1.875 1.875v17.25c0 1.035.84 1.875 1.875 1.875h12.75c1.035 0 1.875-.84 1.875-1.875V12.75A3.75 3.75 0 0 0 16.5 9h-1.875a1.875 1.875 0 0 1-1.875-1.875V5.25A3.75 3.75 0 0 0 9 1.5H5.625ZM7.5 15a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 7.5 15Zm.75 2.25a.75.75 0 0 0 0 1.5H12a.75.75 0 0 0 0-1.5H8.25Z"
-											clip-rule="evenodd"
-										/>
-										<path
-											d="M12.971 1.816A5.23 5.23 0 0 1 14.25 5.25v1.875c0 .207.168.375.375.375H16.5a5.23 5.23 0 0 1 3.434 1.279 9.768 9.768 0 0 0-6.963-6.963Z"
-										/>
-									</svg>
-								{:else if item.type === 'collection'}
-									<svg
-										xmlns="http://www.w3.org/2000/svg"
-										viewBox="0 0 24 24"
-										fill="currentColor"
-										class="size-4"
-									>
-										<path
-											d="M7.5 3.375c0-1.036.84-1.875 1.875-1.875h.375a3.75 3.75 0 0 1 3.75 3.75v1.875C13.5 8.161 14.34 9 15.375 9h1.875A3.75 3.75 0 0 1 21 12.75v3.375C21 17.16 20.16 18 19.125 18h-9.75A1.875 1.875 0 0 1 7.5 16.125V3.375Z"
-										/>
-										<path
-											d="M15 5.25a5.23 5.23 0 0 0-1.279-3.434 9.768 9.768 0 0 1 6.963 6.963A5.23 5.23 0 0 0 17.25 7.5h-1.875A.375.375 0 0 1 15 7.125V5.25ZM4.875 6H6v10.125A3.375 3.375 0 0 0 9.375 19.5H16.5v1.125c0 1.035-.84 1.875-1.875 1.875h-9.75A1.875 1.875 0 0 1 3 20.625V7.875C3 6.839 3.84 6 4.875 6Z"
-										/>
-									</svg>
-								{/if}
-							</div>
-
 							<div class="flex items-center">
 								<div class="flex flex-col">
-									<div
+									<div class=" w-fit mb-0.5">
-										class=" w-fit text-xs font-bold px-1 rounded uppercase line-clamp-1 bg-gray-500/20 text-gray-700 dark:text-gray-200"
+										{#if item.legacy}
-									>
+											<div
-										{item?.type ?? 'Document'}
+												class="bg-gray-500/20 text-gray-700 dark:text-gray-200 rounded uppercase text-xs font-bold px-1"
+											>
+												Legacy
+											</div>
+										{:else if item?.meta?.document}
+											<div
+												class="bg-gray-500/20 text-gray-700 dark:text-gray-200 rounded uppercase text-xs font-bold px-1"
+											>
+												Document
+											</div>
+										{:else}
+											<div
+												class="bg-green-500/20 text-green-700 dark:text-green-200 rounded uppercase text-xs font-bold px-1"
+											>
+												Collection
+											</div>
+										{/if}
 									</div>
 
 									<div class="line-clamp-1 font-medium pr-0.5">

src/lib/stores/index.ts

@@ -29,7 +29,7 @@ export const tags = writable([]);
 
 export const models: Writable<Model[]> = writable([]);
 export const prompts: Writable<Prompt[]> = writable([]);
-export const documents: Writable<Document[]> = writable([]);
+export const knowledge: Writable<Document[]> = writable([]);
 
 export const tools = writable([]);
 export const functions = writable([]);

src/routes/(app)/+layout.svelte

@@ -3,50 +3,46 @@
 	import { onMount, tick, getContext } from 'svelte';
 	import { openDB, deleteDB } from 'idb';
 	import fileSaver from 'file-saver';
+	const { saveAs } = fileSaver;
 	import mermaid from 'mermaid';
 
-	const { saveAs } = fileSaver;
-
 	import { goto } from '$app/navigation';
+	import { page } from '$app/stores';
+	import { fade } from 'svelte/transition';
 
+	import { getKnowledgeItems } from '$lib/apis/knowledge';
+	import { getFunctions } from '$lib/apis/functions';
 	import { getModels as _getModels, getVersionUpdates } from '$lib/apis';
 	import { getAllChatTags } from '$lib/apis/chats';
-
 	import { getPrompts } from '$lib/apis/prompts';
-	import { getDocs } from '$lib/apis/documents';
 	import { getTools } from '$lib/apis/tools';
-
 	import { getBanners } from '$lib/apis/configs';
 	import { getUserSettings } from '$lib/apis/users';
 
-	import {
-		user,
-		showSettings,
-		settings,
-		models,
-		prompts,
-		documents,
-		tags,
-		banners,
-		showChangelog,
-		config,
-		showCallOverlay,
-		tools,
-		functions,
-		temporaryChatEnabled
-	} from '$lib/stores';
-
-	import SettingsModal from '$lib/components/chat/SettingsModal.svelte';
-	import Sidebar from '$lib/components/layout/Sidebar.svelte';
-	import ChangelogModal from '$lib/components/ChangelogModal.svelte';
-	import AccountPending from '$lib/components/layout/Overlay/AccountPending.svelte';
-	import { getFunctions } from '$lib/apis/functions';
-	import { page } from '$app/stores';
 	import { WEBUI_VERSION } from '$lib/constants';
 	import { compareVersion } from '$lib/utils';
 
+	import {
+		config,
+		user,
+		settings,
+		models,
+		prompts,
+		knowledge,
+		tools,
+		functions,
+		tags,
+		banners,
+		showSettings,
+		showChangelog,
+		temporaryChatEnabled
+	} from '$lib/stores';
+
+	import Sidebar from '$lib/components/layout/Sidebar.svelte';
+	import SettingsModal from '$lib/components/chat/SettingsModal.svelte';
+	import ChangelogModal from '$lib/components/ChangelogModal.svelte';
+	import AccountPending from '$lib/components/layout/Overlay/AccountPending.svelte';
 	import UpdateInfoToast from '$lib/components/layout/UpdateInfoToast.svelte';
-	import { fade } from 'svelte/transition';
 
 	const i18n = getContext('i18n');
 
@@ -109,7 +105,7 @@
 					prompts.set(await getPrompts(localStorage.token));
 				})(),
 				(async () => {
-					documents.set(await getDocs(localStorage.token));
+					knowledge.set(await getKnowledgeItems(localStorage.token));
 				})(),
 				(async () => {
 					tools.set(await getTools(localStorage.token));

src/routes/(app)/workspace/+layout.svelte

@@ -61,6 +61,17 @@
 					href="/workspace/models">{$i18n.t('Models')}</a
 				>
 
+				<a
+					class="min-w-fit rounded-lg p-1.5 px-3 {$page.url.pathname.includes(
+						'/workspace/knowledge'
+					)
+						? 'bg-gray-50 dark:bg-gray-850'
+						: ''} transition"
+					href="/workspace/knowledge"
+				>
+					{$i18n.t('Knowledge')}
+				</a>
+
 				<a
 					class="min-w-fit rounded-lg p-1.5 px-3 {$page.url.pathname.includes('/workspace/prompts')
 						? 'bg-gray-50 dark:bg-gray-850'
@@ -68,17 +79,6 @@
 					href="/workspace/prompts">{$i18n.t('Prompts')}</a
 				>
 
-				<a
-					class="min-w-fit rounded-lg p-1.5 px-3 {$page.url.pathname.includes(
-						'/workspace/documents'
-					)
-						? 'bg-gray-50 dark:bg-gray-850'
-						: ''} transition"
-					href="/workspace/documents"
-				>
-					{$i18n.t('Documents')}
-				</a>
-
 				<a
 					class="min-w-fit rounded-lg p-1.5 px-3 {$page.url.pathname.includes('/workspace/tools')
 						? 'bg-gray-50 dark:bg-gray-850'
@@ -101,7 +101,7 @@
 			</div>
 		</div>
 
-		<hr class=" my-2 dark:border-gray-850" />
+		<hr class=" my-2 border-gray-100 dark:border-gray-850" />
 
 		<div class=" py-1 px-5 flex-1 max-h-full overflow-y-auto">
 			<slot />

src/routes/(app)/workspace/documents/+page.svelte

@@ -1,5 +0,0 @@
-<script>
-	import Documents from '$lib/components/workspace/Documents.svelte';
-</script>
-
-<Documents />

src/routes/(app)/workspace/knowledge/+page.svelte

@@ -0,0 +1,5 @@
+<script>
+	import Knowledge from '$lib/components/workspace/Knowledge.svelte';
+</script>
+
+<Knowledge />

src/routes/(app)/workspace/knowledge/[id]/+page.svelte

@@ -0,0 +1,5 @@
+<script>
+	import Collection from '$lib/components/workspace/Knowledge/Collection.svelte';
+</script>
+
+<Collection />

src/routes/(app)/workspace/knowledge/create/+page.svelte

@@ -0,0 +1,5 @@
+<script>
+	import CreateCollection from '$lib/components/workspace/Knowledge/CreateCollection.svelte';
+</script>
+
+<CreateCollection />