From 619c81472b9511525b0769e5fe1d5a0a4d91c95a Mon Sep 17 00:00:00 2001 From: Ethan <92686703+not-a-ethan@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:27:49 -0400 Subject: [PATCH 01/19] Migration link updated --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e83324ead..c8ad50037 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ docker run --rm --volume /var/run/docker.sock:/var/run/docker.sock containrrr/wa In the last part of the command, replace `open-webui` with your container name if it is different. -Check our Migration Guide available in our [Open WebUI Documentation](https://docs.openwebui.com/migration/). +Check our Migration Guide available in our [Open WebUI Documentation](https://docs.openwebui.com/tutorials/migration/). ### Using the Dev Branch 馃寵 From 719f4da1dcdd91d083165aa4956d2a8441d8e665 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Thu, 26 Sep 2024 22:59:09 +0200 Subject: [PATCH 02/19] fix: milvus collection creation issue --- backend/open_webui/apps/rag/vector/dbs/milvus.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/apps/rag/vector/dbs/milvus.py b/backend/open_webui/apps/rag/vector/dbs/milvus.py index 33ec6035a..b4a6a77b3 100644 --- a/backend/open_webui/apps/rag/vector/dbs/milvus.py +++ b/backend/open_webui/apps/rag/vector/dbs/milvus.py @@ -98,7 +98,10 @@ class MilvusClient: index_params = self.client.prepare_index_params() index_params.add_index( - field_name="vector", index_type="HNSW", metric_type="COSINE", params={} + field_name="vector", + index_type="HNSW", + metric_type="COSINE", + params={"M": 16, "efConstruction": 100}, ) self.client.create_collection( From be74a4c9c14612bb82bd1b55a43712f3be773080 Mon Sep 17 00:00:00 2001 From: kivvi Date: Fri, 27 Sep 2024 20:18:13 +0800 Subject: [PATCH 03/19] Fix: O1 does not support the system parameter --- backend/open_webui/apps/openai/main.py | 45 ++++++++++++++------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/backend/open_webui/apps/openai/main.py b/backend/open_webui/apps/openai/main.py index e0a40a1f5..99461b590 100644 --- a/backend/open_webui/apps/openai/main.py +++ b/backend/open_webui/apps/openai/main.py @@ -27,7 +27,6 @@ from fastapi.responses import FileResponse, StreamingResponse from pydantic import BaseModel from starlette.background import BackgroundTask - from open_webui.utils.payload import ( apply_model_params_to_body_openai, apply_model_system_prompt_to_body, @@ -47,7 +46,6 @@ app.add_middleware( allow_headers=["*"], ) - app.state.config = AppConfig() app.state.config.ENABLE_MODEL_FILTER = ENABLE_MODEL_FILTER @@ -193,8 +191,8 @@ async def fetch_url(url, key): async def cleanup_response( - response: Optional[aiohttp.ClientResponse], - session: Optional[aiohttp.ClientSession], + response: Optional[aiohttp.ClientResponse], + session: Optional[aiohttp.ClientSession], ): if response: response.close() @@ -219,18 +217,18 @@ def merge_models_lists(model_lists): } for model in models if "api.openai.com" - not in app.state.config.OPENAI_API_BASE_URLS[idx] - or not any( - name in model["id"] - for name in [ - "babbage", - "dall-e", - "davinci", - "embedding", - "tts", - "whisper", - ] - ) + not in app.state.config.OPENAI_API_BASE_URLS[idx] + or not any( + name in model["id"] + for name in [ + "babbage", + "dall-e", + "davinci", + "embedding", + "tts", + "whisper", + ] + ) ] ) @@ -373,9 +371,9 @@ async def get_models(url_idx: Optional[int] = None, user=Depends(get_verified_us @app.post("/chat/completions") @app.post("/chat/completions/{url_idx}") async def generate_chat_completion( - form_data: dict, - url_idx: Optional[int] = None, - user=Depends(get_verified_user), + form_data: dict, + url_idx: Optional[int] = None, + user=Depends(get_verified_user), ): idx = 0 payload = {**form_data} @@ -407,20 +405,25 @@ async def generate_chat_completion( url = app.state.config.OPENAI_API_BASE_URLS[idx] key = app.state.config.OPENAI_API_KEYS[idx] + is_o1 = payload["model"].lower().startswith("o1") # Change max_completion_tokens to max_tokens (Backward compatible) - if "api.openai.com" not in url and not payload["model"].lower().startswith("o1-"): + if "api.openai.com" not in url and not is_o1: if "max_completion_tokens" in payload: # Remove "max_completion_tokens" from the payload payload["max_tokens"] = payload["max_completion_tokens"] del payload["max_completion_tokens"] else: - if payload["model"].lower().startswith("o1-") and "max_tokens" in payload: + if is_o1 and "max_tokens" in payload: payload["max_completion_tokens"] = payload["max_tokens"] del payload["max_tokens"] if "max_tokens" in payload and "max_completion_tokens" in payload: del payload["max_tokens"] + # Fix: O1 does not support the "system" parameter, Modify "system" to "user" + if is_o1 and payload["messages"][0]["role"] == "system": + payload["messages"][0]["role"] = "user" + # Convert the modified body back to JSON payload = json.dumps(payload) From 0bd9d59c78213ad2b3cd9c1440bef5ca5d619edc Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 27 Sep 2024 14:38:56 +0200 Subject: [PATCH 04/19] refac: update check timeout --- backend/open_webui/main.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 4af48906b..ca829cfa1 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -2153,7 +2153,8 @@ async def get_app_changelog(): @app.get("/api/version/updates") async def get_app_latest_release_version(): try: - async with aiohttp.ClientSession(trust_env=True) as session: + timeout = aiohttp.ClientTimeout(total=1) + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: async with session.get( "https://api.github.com/repos/open-webui/open-webui/releases/latest" ) as response: @@ -2163,10 +2164,7 @@ async def get_app_latest_release_version(): return {"current": VERSION, "latest": latest_version[1:]} except aiohttp.ClientError: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail=ERROR_MESSAGES.RATE_LIMIT_EXCEEDED, - ) + return {"current": VERSION, "latest": VERSION} ############################ From 44d768ecf3b53055fa249bf9a7031f80b93d5887 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 27 Sep 2024 14:41:29 +0200 Subject: [PATCH 05/19] refac: do not wait for update check --- src/routes/(app)/+layout.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/(app)/+layout.svelte b/src/routes/(app)/+layout.svelte index ad2f085cf..83a53dffd 100644 --- a/src/routes/(app)/+layout.svelte +++ b/src/routes/(app)/+layout.svelte @@ -206,10 +206,10 @@ const now = new Date(); if (now - dismissedUpdateToast > 24 * 60 * 60 * 1000) { - await checkForVersionUpdates(); + checkForVersionUpdates(); } } else { - await checkForVersionUpdates(); + checkForVersionUpdates(); } } await tick(); From 464b6a329edbb353a0b7a51efd1ebbc31a7a092d Mon Sep 17 00:00:00 2001 From: Aleix Dorca Date: Fri, 27 Sep 2024 16:11:52 +0200 Subject: [PATCH 06/19] Update catalan translation.json --- src/lib/i18n/locales/ca-ES/translation.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/i18n/locales/ca-ES/translation.json b/src/lib/i18n/locales/ca-ES/translation.json index 25907c0f2..bd6179558 100644 --- a/src/lib/i18n/locales/ca-ES/translation.json +++ b/src/lib/i18n/locales/ca-ES/translation.json @@ -9,7 +9,7 @@ "{{user}}'s Chats": "Els xats de {{user}}", "{{webUIName}} Backend Required": "El Backend de {{webUIName}} 茅s necessari", "*Prompt node ID(s) are required for image generation": "*Els identificadors de nodes d'indicacions s贸n necessaris per a la generaci贸 d'imatges", - "A new version (v{{LATEST_VERSION}}) is now available.": "", + "A new version (v{{LATEST_VERSION}}) is now available.": "Hi ha una nova versi贸 disponible (v{{LATEST_VERSION}}).", "A task model is used when performing tasks such as generating titles for chats and web search queries": "Un model de tasca s'utilitza quan es realitzen tasques com ara generar t铆tols per a xats i consultes de cerca per a la web", "a user": "un usuari", "About": "Sobre", @@ -466,7 +466,7 @@ "Oops! Looks like the URL is invalid. Please double-check and try again.": "Ui! Sembla que l'URL no 茅s v脿lida. Si us plau, revisa-la i torna-ho a provar.", "Oops! There was an error in the previous response. Please try again or contact admin.": "Ui! Hi ha hagut un error en la resposta anterior. Torna a provar-ho o contacta amb un administrador", "Oops! You're using an unsupported method (frontend only). Please serve the WebUI from the backend.": "Ui! Est脿s utilitzant un m猫tode no suportat (nom茅s frontend). Si us plau, serveix la WebUI des del backend.", - "Open file": "", + "Open file": "Obrir arxiu", "Open new chat": "Obre un xat nou", "Open WebUI version (v{{OPEN_WEBUI_VERSION}}) is lower than required version (v{{REQUIRED_VERSION}})": "La versi贸 d'Open WebUI (v{{OPEN_WEBUI_VERSION}}) 茅s inferior a la versi贸 requerida (v{{REQUIRED_VERSION}})", "OpenAI": "OpenAI", @@ -478,7 +478,7 @@ "Other": "Altres", "Output format": "Format de sortida", "Overview": "Vista general", - "page": "", + "page": "p脿gina", "Password": "Contrasenya", "PDF document (.pdf)": "Document PDF (.pdf)", "PDF Extract Images (OCR)": "Extreu imatges del PDF (OCR)", @@ -497,7 +497,7 @@ "Plain text (.txt)": "Text pla (.txt)", "Playground": "Zona de jocs", "Please carefully review the following warnings:": "Si us plau, revisa els seg眉ents avisos amb cura:", - "Please select a reason": "", + "Please select a reason": "Si us plau, selecciona una ra贸", "Positive attitude": "Actitud positiva", "Previous 30 days": "30 dies anteriors", "Previous 7 days": "7 dies anteriors", @@ -704,7 +704,7 @@ "Unpin": "Alliberar", "Update": "Actualitzar", "Update and Copy Link": "Actualitzar i copiar l'enlla莽", - "Update for the latest features and improvements.": "", + "Update for the latest features and improvements.": "Actualitza per a les darreres caracter铆stiques i millores.", "Update password": "Actualitzar la contrasenya", "Updated at": "Actualitzat", "Upload": "Pujar", From e13614e11bcd50ecd206411b5e49ec25520d0865 Mon Sep 17 00:00:00 2001 From: kivvi Date: Fri, 27 Sep 2024 20:18:13 +0800 Subject: [PATCH 07/19] Fix: O1 does not support the system parameter --- backend/open_webui/apps/openai/main.py | 45 ++++++++++++++------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/backend/open_webui/apps/openai/main.py b/backend/open_webui/apps/openai/main.py index e0a40a1f5..5768fe645 100644 --- a/backend/open_webui/apps/openai/main.py +++ b/backend/open_webui/apps/openai/main.py @@ -27,7 +27,6 @@ from fastapi.responses import FileResponse, StreamingResponse from pydantic import BaseModel from starlette.background import BackgroundTask - from open_webui.utils.payload import ( apply_model_params_to_body_openai, apply_model_system_prompt_to_body, @@ -47,7 +46,6 @@ app.add_middleware( allow_headers=["*"], ) - app.state.config = AppConfig() app.state.config.ENABLE_MODEL_FILTER = ENABLE_MODEL_FILTER @@ -193,8 +191,8 @@ async def fetch_url(url, key): async def cleanup_response( - response: Optional[aiohttp.ClientResponse], - session: Optional[aiohttp.ClientSession], + response: Optional[aiohttp.ClientResponse], + session: Optional[aiohttp.ClientSession], ): if response: response.close() @@ -219,18 +217,18 @@ def merge_models_lists(model_lists): } for model in models if "api.openai.com" - not in app.state.config.OPENAI_API_BASE_URLS[idx] - or not any( - name in model["id"] - for name in [ - "babbage", - "dall-e", - "davinci", - "embedding", - "tts", - "whisper", - ] - ) + not in app.state.config.OPENAI_API_BASE_URLS[idx] + or not any( + name in model["id"] + for name in [ + "babbage", + "dall-e", + "davinci", + "embedding", + "tts", + "whisper", + ] + ) ] ) @@ -373,9 +371,9 @@ async def get_models(url_idx: Optional[int] = None, user=Depends(get_verified_us @app.post("/chat/completions") @app.post("/chat/completions/{url_idx}") async def generate_chat_completion( - form_data: dict, - url_idx: Optional[int] = None, - user=Depends(get_verified_user), + form_data: dict, + url_idx: Optional[int] = None, + user=Depends(get_verified_user), ): idx = 0 payload = {**form_data} @@ -407,20 +405,25 @@ async def generate_chat_completion( url = app.state.config.OPENAI_API_BASE_URLS[idx] key = app.state.config.OPENAI_API_KEYS[idx] + is_o1 = payload["model"].lower().startswith("o1-") # Change max_completion_tokens to max_tokens (Backward compatible) - if "api.openai.com" not in url and not payload["model"].lower().startswith("o1-"): + if "api.openai.com" not in url and not is_o1: if "max_completion_tokens" in payload: # Remove "max_completion_tokens" from the payload payload["max_tokens"] = payload["max_completion_tokens"] del payload["max_completion_tokens"] else: - if payload["model"].lower().startswith("o1-") and "max_tokens" in payload: + if is_o1 and "max_tokens" in payload: payload["max_completion_tokens"] = payload["max_tokens"] del payload["max_tokens"] if "max_tokens" in payload and "max_completion_tokens" in payload: del payload["max_tokens"] + # Fix: O1 does not support the "system" parameter, Modify "system" to "user" + if is_o1 and payload["messages"][0]["role"] == "system": + payload["messages"][0]["role"] = "user" + # Convert the modified body back to JSON payload = json.dumps(payload) From 4ead3c5b8059a55fa65f22b26c5763da0b1d5809 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 27 Sep 2024 19:43:40 +0200 Subject: [PATCH 08/19] chore: format --- backend/open_webui/apps/openai/main.py | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/backend/open_webui/apps/openai/main.py b/backend/open_webui/apps/openai/main.py index 5768fe645..9d62f32d2 100644 --- a/backend/open_webui/apps/openai/main.py +++ b/backend/open_webui/apps/openai/main.py @@ -191,8 +191,8 @@ async def fetch_url(url, key): async def cleanup_response( - response: Optional[aiohttp.ClientResponse], - session: Optional[aiohttp.ClientSession], + response: Optional[aiohttp.ClientResponse], + session: Optional[aiohttp.ClientSession], ): if response: response.close() @@ -217,18 +217,18 @@ def merge_models_lists(model_lists): } for model in models if "api.openai.com" - not in app.state.config.OPENAI_API_BASE_URLS[idx] - or not any( - name in model["id"] - for name in [ - "babbage", - "dall-e", - "davinci", - "embedding", - "tts", - "whisper", - ] - ) + not in app.state.config.OPENAI_API_BASE_URLS[idx] + or not any( + name in model["id"] + for name in [ + "babbage", + "dall-e", + "davinci", + "embedding", + "tts", + "whisper", + ] + ) ] ) @@ -371,9 +371,9 @@ async def get_models(url_idx: Optional[int] = None, user=Depends(get_verified_us @app.post("/chat/completions") @app.post("/chat/completions/{url_idx}") async def generate_chat_completion( - form_data: dict, - url_idx: Optional[int] = None, - user=Depends(get_verified_user), + form_data: dict, + url_idx: Optional[int] = None, + user=Depends(get_verified_user), ): idx = 0 payload = {**form_data} From eab30781e0e02033c66df1f659118204ab3bbc98 Mon Sep 17 00:00:00 2001 From: Hugo Haldi <20846785+HaldiH@users.noreply.github.com> Date: Fri, 27 Sep 2024 20:04:45 +0200 Subject: [PATCH 09/19] Chat completion 401 when no Authorization header When we send a request to `/api/chat/completions` without the `Authorization` header, the server just crashes and creates a stack trace, returning "Internal Server Error" to the calling client. With this fix, the server sends a 401 to the client with the content `{"detail": "Not authenticated"}`. --- backend/open_webui/main.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 4af48906b..9c075f367 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -761,10 +761,22 @@ class PipelineMiddleware(BaseHTTPMiddleware): # Parse string to JSON data = json.loads(body_str) if body_str else {} - user = get_current_user( - request, - get_http_authorization_cred(request.headers["Authorization"]), - ) + try: + user = get_current_user( + request, + get_http_authorization_cred(request.headers["Authorization"]), + ) + except KeyError as e: + if len(e.args) > 1: + return JSONResponse( + status_code=e.args[0], + content={"detail": e.args[1]}, + ) + else: + return JSONResponse( + status_code=status.HTTP_401_UNAUTHORIZED, + content={"detail": "Not authenticated"}, + ) try: data = filter_pipeline(data, user) From e1103305f5466dd57767d0f6bcb388839b0a98b3 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 01:27:46 +0200 Subject: [PATCH 10/19] refac: "rag" endpoints renamed to "retrieval" --- .../apps/{rag => retrieval}/main.py | 3 +- .../apps/{rag => retrieval}/search/brave.py | 0 .../{rag => retrieval}/search/duckduckgo.py | 0 .../{rag => retrieval}/search/google_pse.py | 0 .../{rag => retrieval}/search/jina_search.py | 0 .../apps/{rag => retrieval}/search/main.py | 0 .../{rag => retrieval}/search/searchapi.py | 0 .../apps/{rag => retrieval}/search/searxng.py | 0 .../apps/{rag => retrieval}/search/serper.py | 0 .../apps/{rag => retrieval}/search/serply.py | 0 .../{rag => retrieval}/search/serpstack.py | 0 .../apps/{rag => retrieval}/search/tavily.py | 0 .../search/testdata/brave.json | 0 .../search/testdata/google_pse.json | 0 .../search/testdata/searchapi.json | 0 .../search/testdata/searxng.json | 0 .../search/testdata/serper.json | 0 .../search/testdata/serply.json | 0 .../search/testdata/serpstack.json | 0 .../apps/{rag => retrieval}/utils.py | 0 .../{rag => retrieval}/vector/connector.py | 0 .../{rag => retrieval}/vector/dbs/chroma.py | 0 .../{rag => retrieval}/vector/dbs/milvus.py | 0 .../apps/{rag => retrieval}/vector/main.py | 0 backend/open_webui/main.py | 65 +++++++++++-------- src/lib/components/chat/MessageInput.svelte | 5 +- src/lib/constants.ts | 2 +- 27 files changed, 41 insertions(+), 34 deletions(-) rename backend/open_webui/apps/{rag => retrieval}/main.py (99%) rename backend/open_webui/apps/{rag => retrieval}/search/brave.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/duckduckgo.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/google_pse.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/jina_search.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/main.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/searchapi.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/searxng.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/serper.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/serply.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/serpstack.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/tavily.py (100%) rename backend/open_webui/apps/{rag => retrieval}/search/testdata/brave.json (100%) rename backend/open_webui/apps/{rag => retrieval}/search/testdata/google_pse.json (100%) rename backend/open_webui/apps/{rag => retrieval}/search/testdata/searchapi.json (100%) rename backend/open_webui/apps/{rag => retrieval}/search/testdata/searxng.json (100%) rename backend/open_webui/apps/{rag => retrieval}/search/testdata/serper.json (100%) rename backend/open_webui/apps/{rag => retrieval}/search/testdata/serply.json (100%) rename backend/open_webui/apps/{rag => retrieval}/search/testdata/serpstack.json (100%) rename backend/open_webui/apps/{rag => retrieval}/utils.py (100%) rename backend/open_webui/apps/{rag => retrieval}/vector/connector.py (100%) rename backend/open_webui/apps/{rag => retrieval}/vector/dbs/chroma.py (100%) rename backend/open_webui/apps/{rag => retrieval}/vector/dbs/milvus.py (100%) rename backend/open_webui/apps/{rag => retrieval}/vector/main.py (100%) diff --git a/backend/open_webui/apps/rag/main.py b/backend/open_webui/apps/retrieval/main.py similarity index 99% rename from backend/open_webui/apps/rag/main.py rename to backend/open_webui/apps/retrieval/main.py index 7b476c056..d276bd80b 100644 --- a/backend/open_webui/apps/rag/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -1061,7 +1061,7 @@ def store_data_in_vector_db( if len(docs) > 0: log.info(f"store_data_in_vector_db {docs}") - return store_docs_in_vector_db(docs, collection_name, metadata, overwrite), None + return store_docs_in_vector_db(docs, collection_name, metadata, overwrite) else: raise ValueError(ERROR_MESSAGES.EMPTY_CONTENT) @@ -1377,6 +1377,7 @@ def process_doc( ) if result: + return { "status": True, "collection_name": collection_name, diff --git a/backend/open_webui/apps/rag/search/brave.py b/backend/open_webui/apps/retrieval/search/brave.py similarity index 100% rename from backend/open_webui/apps/rag/search/brave.py rename to backend/open_webui/apps/retrieval/search/brave.py diff --git a/backend/open_webui/apps/rag/search/duckduckgo.py b/backend/open_webui/apps/retrieval/search/duckduckgo.py similarity index 100% rename from backend/open_webui/apps/rag/search/duckduckgo.py rename to backend/open_webui/apps/retrieval/search/duckduckgo.py diff --git a/backend/open_webui/apps/rag/search/google_pse.py b/backend/open_webui/apps/retrieval/search/google_pse.py similarity index 100% rename from backend/open_webui/apps/rag/search/google_pse.py rename to backend/open_webui/apps/retrieval/search/google_pse.py diff --git a/backend/open_webui/apps/rag/search/jina_search.py b/backend/open_webui/apps/retrieval/search/jina_search.py similarity index 100% rename from backend/open_webui/apps/rag/search/jina_search.py rename to backend/open_webui/apps/retrieval/search/jina_search.py diff --git a/backend/open_webui/apps/rag/search/main.py b/backend/open_webui/apps/retrieval/search/main.py similarity index 100% rename from backend/open_webui/apps/rag/search/main.py rename to backend/open_webui/apps/retrieval/search/main.py diff --git a/backend/open_webui/apps/rag/search/searchapi.py b/backend/open_webui/apps/retrieval/search/searchapi.py similarity index 100% rename from backend/open_webui/apps/rag/search/searchapi.py rename to backend/open_webui/apps/retrieval/search/searchapi.py diff --git a/backend/open_webui/apps/rag/search/searxng.py b/backend/open_webui/apps/retrieval/search/searxng.py similarity index 100% rename from backend/open_webui/apps/rag/search/searxng.py rename to backend/open_webui/apps/retrieval/search/searxng.py diff --git a/backend/open_webui/apps/rag/search/serper.py b/backend/open_webui/apps/retrieval/search/serper.py similarity index 100% rename from backend/open_webui/apps/rag/search/serper.py rename to backend/open_webui/apps/retrieval/search/serper.py diff --git a/backend/open_webui/apps/rag/search/serply.py b/backend/open_webui/apps/retrieval/search/serply.py similarity index 100% rename from backend/open_webui/apps/rag/search/serply.py rename to backend/open_webui/apps/retrieval/search/serply.py diff --git a/backend/open_webui/apps/rag/search/serpstack.py b/backend/open_webui/apps/retrieval/search/serpstack.py similarity index 100% rename from backend/open_webui/apps/rag/search/serpstack.py rename to backend/open_webui/apps/retrieval/search/serpstack.py diff --git a/backend/open_webui/apps/rag/search/tavily.py b/backend/open_webui/apps/retrieval/search/tavily.py similarity index 100% rename from backend/open_webui/apps/rag/search/tavily.py rename to backend/open_webui/apps/retrieval/search/tavily.py diff --git a/backend/open_webui/apps/rag/search/testdata/brave.json b/backend/open_webui/apps/retrieval/search/testdata/brave.json similarity index 100% rename from backend/open_webui/apps/rag/search/testdata/brave.json rename to backend/open_webui/apps/retrieval/search/testdata/brave.json diff --git a/backend/open_webui/apps/rag/search/testdata/google_pse.json b/backend/open_webui/apps/retrieval/search/testdata/google_pse.json similarity index 100% rename from backend/open_webui/apps/rag/search/testdata/google_pse.json rename to backend/open_webui/apps/retrieval/search/testdata/google_pse.json diff --git a/backend/open_webui/apps/rag/search/testdata/searchapi.json b/backend/open_webui/apps/retrieval/search/testdata/searchapi.json similarity index 100% rename from backend/open_webui/apps/rag/search/testdata/searchapi.json rename to backend/open_webui/apps/retrieval/search/testdata/searchapi.json diff --git a/backend/open_webui/apps/rag/search/testdata/searxng.json b/backend/open_webui/apps/retrieval/search/testdata/searxng.json similarity index 100% rename from backend/open_webui/apps/rag/search/testdata/searxng.json rename to backend/open_webui/apps/retrieval/search/testdata/searxng.json diff --git a/backend/open_webui/apps/rag/search/testdata/serper.json b/backend/open_webui/apps/retrieval/search/testdata/serper.json similarity index 100% rename from backend/open_webui/apps/rag/search/testdata/serper.json rename to backend/open_webui/apps/retrieval/search/testdata/serper.json diff --git a/backend/open_webui/apps/rag/search/testdata/serply.json b/backend/open_webui/apps/retrieval/search/testdata/serply.json similarity index 100% rename from backend/open_webui/apps/rag/search/testdata/serply.json rename to backend/open_webui/apps/retrieval/search/testdata/serply.json diff --git a/backend/open_webui/apps/rag/search/testdata/serpstack.json b/backend/open_webui/apps/retrieval/search/testdata/serpstack.json similarity index 100% rename from backend/open_webui/apps/rag/search/testdata/serpstack.json rename to backend/open_webui/apps/retrieval/search/testdata/serpstack.json diff --git a/backend/open_webui/apps/rag/utils.py b/backend/open_webui/apps/retrieval/utils.py similarity index 100% rename from backend/open_webui/apps/rag/utils.py rename to backend/open_webui/apps/retrieval/utils.py diff --git a/backend/open_webui/apps/rag/vector/connector.py b/backend/open_webui/apps/retrieval/vector/connector.py similarity index 100% rename from backend/open_webui/apps/rag/vector/connector.py rename to backend/open_webui/apps/retrieval/vector/connector.py diff --git a/backend/open_webui/apps/rag/vector/dbs/chroma.py b/backend/open_webui/apps/retrieval/vector/dbs/chroma.py similarity index 100% rename from backend/open_webui/apps/rag/vector/dbs/chroma.py rename to backend/open_webui/apps/retrieval/vector/dbs/chroma.py diff --git a/backend/open_webui/apps/rag/vector/dbs/milvus.py b/backend/open_webui/apps/retrieval/vector/dbs/milvus.py similarity index 100% rename from backend/open_webui/apps/rag/vector/dbs/milvus.py rename to backend/open_webui/apps/retrieval/vector/dbs/milvus.py diff --git a/backend/open_webui/apps/rag/vector/main.py b/backend/open_webui/apps/retrieval/vector/main.py similarity index 100% rename from backend/open_webui/apps/rag/vector/main.py rename to backend/open_webui/apps/retrieval/vector/main.py diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index b6fa63fc3..4c1a2053d 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -16,37 +16,45 @@ from typing import Optional import aiohttp import requests - -from open_webui.apps.audio.main import app as audio_app -from open_webui.apps.images.main import app as images_app -from open_webui.apps.ollama.main import app as ollama_app from open_webui.apps.ollama.main import ( - GenerateChatCompletionForm, + app as ollama_app, + get_all_models as get_ollama_models, generate_chat_completion as generate_ollama_chat_completion, generate_openai_chat_completion as generate_ollama_openai_chat_completion, + GenerateChatCompletionForm, ) -from open_webui.apps.ollama.main import get_all_models as get_ollama_models -from open_webui.apps.openai.main import app as openai_app from open_webui.apps.openai.main import ( + app as openai_app, generate_chat_completion as generate_openai_chat_completion, + get_all_models as get_openai_models, ) -from open_webui.apps.openai.main import get_all_models as get_openai_models -from open_webui.apps.rag.main import app as rag_app -from open_webui.apps.rag.utils import get_rag_context, rag_template -from open_webui.apps.socket.main import app as socket_app, periodic_usage_pool_cleanup -from open_webui.apps.socket.main import get_event_call, get_event_emitter -from open_webui.apps.webui.internal.db import Session -from open_webui.apps.webui.main import app as webui_app + +from open_webui.apps.retrieval.main import app as retrieval_app +from open_webui.apps.retrieval.utils import get_rag_context, rag_template + +from open_webui.apps.socket.main import ( + app as socket_app, + periodic_usage_pool_cleanup, + get_event_call, + get_event_emitter, +) + from open_webui.apps.webui.main import ( + app as webui_app, generate_function_chat_completion, get_pipe_models, ) +from open_webui.apps.webui.internal.db import Session + from open_webui.apps.webui.models.auths import Auths from open_webui.apps.webui.models.functions import Functions from open_webui.apps.webui.models.models import Models from open_webui.apps.webui.models.users import UserModel, Users + from open_webui.apps.webui.utils import load_function_module_by_id +from open_webui.apps.audio.main import app as audio_app +from open_webui.apps.images.main import app as images_app from authlib.integrations.starlette_client import OAuth from authlib.oidc.core import UserInfo @@ -491,11 +499,11 @@ async def chat_completion_files_handler(body) -> tuple[dict, dict[str, list]]: contexts, citations = get_rag_context( files=files, messages=body["messages"], - embedding_function=rag_app.state.EMBEDDING_FUNCTION, - k=rag_app.state.config.TOP_K, - reranking_function=rag_app.state.sentence_transformer_rf, - r=rag_app.state.config.RELEVANCE_THRESHOLD, - hybrid_search=rag_app.state.config.ENABLE_RAG_HYBRID_SEARCH, + embedding_function=retrieval_app.state.EMBEDDING_FUNCTION, + k=retrieval_app.state.config.TOP_K, + reranking_function=retrieval_app.state.sentence_transformer_rf, + r=retrieval_app.state.config.RELEVANCE_THRESHOLD, + hybrid_search=retrieval_app.state.config.ENABLE_RAG_HYBRID_SEARCH, ) log.debug(f"rag_contexts: {contexts}, citations: {citations}") @@ -608,7 +616,7 @@ class ChatCompletionMiddleware(BaseHTTPMiddleware): if prompt is None: raise Exception("No user message found") if ( - rag_app.state.config.RELEVANCE_THRESHOLD == 0 + retrieval_app.state.config.RELEVANCE_THRESHOLD == 0 and context_string.strip() == "" ): log.debug( @@ -620,14 +628,14 @@ class ChatCompletionMiddleware(BaseHTTPMiddleware): if model["owned_by"] == "ollama": body["messages"] = prepend_to_first_user_message_content( rag_template( - rag_app.state.config.RAG_TEMPLATE, context_string, prompt + retrieval_app.state.config.RAG_TEMPLATE, context_string, prompt ), body["messages"], ) else: body["messages"] = add_or_update_system_message( rag_template( - rag_app.state.config.RAG_TEMPLATE, context_string, prompt + retrieval_app.state.config.RAG_TEMPLATE, context_string, prompt ), body["messages"], ) @@ -849,7 +857,7 @@ async def check_url(request: Request, call_next): async def update_embedding_function(request: Request, call_next): response = await call_next(request) if "/embedding/update" in request.url.path: - webui_app.state.EMBEDDING_FUNCTION = rag_app.state.EMBEDDING_FUNCTION + webui_app.state.EMBEDDING_FUNCTION = retrieval_app.state.EMBEDDING_FUNCTION return response @@ -877,11 +885,12 @@ app.mount("/openai", openai_app) app.mount("/images/api/v1", images_app) app.mount("/audio/api/v1", audio_app) -app.mount("/rag/api/v1", rag_app) +app.mount("/retrieval/api/v1", retrieval_app) app.mount("/api/v1", webui_app) -webui_app.state.EMBEDDING_FUNCTION = rag_app.state.EMBEDDING_FUNCTION + +webui_app.state.EMBEDDING_FUNCTION = retrieval_app.state.EMBEDDING_FUNCTION async def get_all_models(): @@ -2066,7 +2075,7 @@ async def get_app_config(request: Request): "enable_login_form": webui_app.state.config.ENABLE_LOGIN_FORM, **( { - "enable_web_search": rag_app.state.config.ENABLE_RAG_WEB_SEARCH, + "enable_web_search": retrieval_app.state.config.ENABLE_RAG_WEB_SEARCH, "enable_image_generation": images_app.state.config.ENABLED, "enable_community_sharing": webui_app.state.config.ENABLE_COMMUNITY_SHARING, "enable_message_rating": webui_app.state.config.ENABLE_MESSAGE_RATING, @@ -2092,8 +2101,8 @@ async def get_app_config(request: Request): }, }, "file": { - "max_size": rag_app.state.config.FILE_MAX_SIZE, - "max_count": rag_app.state.config.FILE_MAX_COUNT, + "max_size": retrieval_app.state.config.FILE_MAX_SIZE, + "max_count": retrieval_app.state.config.FILE_MAX_COUNT, }, "permissions": {**webui_app.state.config.USER_PERMISSIONS}, } diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index ea6b0aec8..0a10d5393 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -159,16 +159,13 @@ const processFileItem = async (fileItem) => { try { const res = await processDocToVectorDB(localStorage.token, fileItem.id); - if (res) { fileItem.status = 'processed'; fileItem.collection_name = res.collection_name; files = files; } } catch (e) { - // Remove the failed doc from the files array - // files = files.filter((f) => f.id !== fileItem.id); - toast.error(e); + // We keep the file in the files list even if it fails to process fileItem.status = 'processed'; files = files; } diff --git a/src/lib/constants.ts b/src/lib/constants.ts index ad7b5c29e..8820c0d99 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -11,7 +11,7 @@ export const OLLAMA_API_BASE_URL = `${WEBUI_BASE_URL}/ollama`; export const OPENAI_API_BASE_URL = `${WEBUI_BASE_URL}/openai`; export const AUDIO_API_BASE_URL = `${WEBUI_BASE_URL}/audio/api/v1`; export const IMAGES_API_BASE_URL = `${WEBUI_BASE_URL}/images/api/v1`; -export const RAG_API_BASE_URL = `${WEBUI_BASE_URL}/rag/api/v1`; +export const RAG_API_BASE_URL = `${WEBUI_BASE_URL}/retrieval/api/v1`; export const WEBUI_VERSION = APP_VERSION; export const WEBUI_BUILD_HASH = APP_BUILD_HASH; From 5b7cf889153dbc6f81b49364d19d15622af40400 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 01:28:45 +0200 Subject: [PATCH 11/19] refac --- backend/open_webui/apps/retrieval/main.py | 26 +++++++++---------- .../open_webui/apps/retrieval/search/brave.py | 2 +- .../apps/retrieval/search/duckduckgo.py | 2 +- .../apps/retrieval/search/google_pse.py | 2 +- .../apps/retrieval/search/jina_search.py | 2 +- .../apps/retrieval/search/searchapi.py | 2 +- .../apps/retrieval/search/searxng.py | 2 +- .../apps/retrieval/search/serper.py | 2 +- .../apps/retrieval/search/serply.py | 2 +- .../apps/retrieval/search/serpstack.py | 2 +- .../apps/retrieval/search/tavily.py | 2 +- backend/open_webui/apps/retrieval/utils.py | 2 +- .../apps/retrieval/vector/connector.py | 4 +-- .../apps/retrieval/vector/dbs/chroma.py | 2 +- .../apps/retrieval/vector/dbs/milvus.py | 2 +- .../open_webui/apps/webui/routers/memories.py | 2 +- 16 files changed, 29 insertions(+), 29 deletions(-) diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index d276bd80b..e4f98b8b0 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -20,18 +20,18 @@ from fastapi import Depends, FastAPI, File, Form, HTTPException, UploadFile, sta from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel -from open_webui.apps.rag.search.main import SearchResult -from open_webui.apps.rag.search.brave import search_brave -from open_webui.apps.rag.search.duckduckgo import search_duckduckgo -from open_webui.apps.rag.search.google_pse import search_google_pse -from open_webui.apps.rag.search.jina_search import search_jina -from open_webui.apps.rag.search.searchapi import search_searchapi -from open_webui.apps.rag.search.searxng import search_searxng -from open_webui.apps.rag.search.serper import search_serper -from open_webui.apps.rag.search.serply import search_serply -from open_webui.apps.rag.search.serpstack import search_serpstack -from open_webui.apps.rag.search.tavily import search_tavily -from open_webui.apps.rag.utils import ( +from open_webui.apps.retrieval.search.main import SearchResult +from open_webui.apps.retrieval.search.brave import search_brave +from open_webui.apps.retrieval.search.duckduckgo import search_duckduckgo +from open_webui.apps.retrieval.search.google_pse import search_google_pse +from open_webui.apps.retrieval.search.jina_search import search_jina +from open_webui.apps.retrieval.search.searchapi import search_searchapi +from open_webui.apps.retrieval.search.searxng import search_searxng +from open_webui.apps.retrieval.search.serper import search_serper +from open_webui.apps.retrieval.search.serply import search_serply +from open_webui.apps.retrieval.search.serpstack import search_serpstack +from open_webui.apps.retrieval.search.tavily import search_tavily +from open_webui.apps.retrieval.utils import ( get_embedding_function, get_model_path, query_collection, @@ -98,7 +98,7 @@ from open_webui.utils.misc import ( sanitize_filename, ) from open_webui.utils.utils import get_admin_user, get_verified_user -from open_webui.apps.rag.vector.connector import VECTOR_DB_CLIENT +from open_webui.apps.retrieval.vector.connector import VECTOR_DB_CLIENT from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.document_loaders import ( diff --git a/backend/open_webui/apps/retrieval/search/brave.py b/backend/open_webui/apps/retrieval/search/brave.py index 2eb256b4b..11a2938b2 100644 --- a/backend/open_webui/apps/retrieval/search/brave.py +++ b/backend/open_webui/apps/retrieval/search/brave.py @@ -2,7 +2,7 @@ import logging from typing import Optional import requests -from open_webui.apps.rag.search.main import SearchResult, get_filtered_results +from open_webui.apps.retrieval.search.main import SearchResult, get_filtered_results from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) diff --git a/backend/open_webui/apps/retrieval/search/duckduckgo.py b/backend/open_webui/apps/retrieval/search/duckduckgo.py index a8a580aca..82558ba37 100644 --- a/backend/open_webui/apps/retrieval/search/duckduckgo.py +++ b/backend/open_webui/apps/retrieval/search/duckduckgo.py @@ -1,7 +1,7 @@ import logging from typing import Optional -from open_webui.apps.rag.search.main import SearchResult, get_filtered_results +from open_webui.apps.retrieval.search.main import SearchResult, get_filtered_results from duckduckgo_search import DDGS from open_webui.env import SRC_LOG_LEVELS diff --git a/backend/open_webui/apps/retrieval/search/google_pse.py b/backend/open_webui/apps/retrieval/search/google_pse.py index a7f75a6c6..c42851f47 100644 --- a/backend/open_webui/apps/retrieval/search/google_pse.py +++ b/backend/open_webui/apps/retrieval/search/google_pse.py @@ -2,7 +2,7 @@ import logging from typing import Optional import requests -from open_webui.apps.rag.search.main import SearchResult, get_filtered_results +from open_webui.apps.retrieval.search.main import SearchResult, get_filtered_results from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) diff --git a/backend/open_webui/apps/retrieval/search/jina_search.py b/backend/open_webui/apps/retrieval/search/jina_search.py index 41cde679d..f44f10d5c 100644 --- a/backend/open_webui/apps/retrieval/search/jina_search.py +++ b/backend/open_webui/apps/retrieval/search/jina_search.py @@ -1,7 +1,7 @@ import logging import requests -from open_webui.apps.rag.search.main import SearchResult +from open_webui.apps.retrieval.search.main import SearchResult from open_webui.env import SRC_LOG_LEVELS from yarl import URL diff --git a/backend/open_webui/apps/retrieval/search/searchapi.py b/backend/open_webui/apps/retrieval/search/searchapi.py index 9ec9a0747..a648d6600 100644 --- a/backend/open_webui/apps/retrieval/search/searchapi.py +++ b/backend/open_webui/apps/retrieval/search/searchapi.py @@ -3,7 +3,7 @@ from typing import Optional from urllib.parse import urlencode import requests -from open_webui.apps.rag.search.main import SearchResult, get_filtered_results +from open_webui.apps.retrieval.search.main import SearchResult, get_filtered_results from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) diff --git a/backend/open_webui/apps/retrieval/search/searxng.py b/backend/open_webui/apps/retrieval/search/searxng.py index 26c534aa3..14b6b40b5 100644 --- a/backend/open_webui/apps/retrieval/search/searxng.py +++ b/backend/open_webui/apps/retrieval/search/searxng.py @@ -2,7 +2,7 @@ import logging from typing import Optional import requests -from open_webui.apps.rag.search.main import SearchResult, get_filtered_results +from open_webui.apps.retrieval.search.main import SearchResult, get_filtered_results from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) diff --git a/backend/open_webui/apps/retrieval/search/serper.py b/backend/open_webui/apps/retrieval/search/serper.py index ed7cc2c5f..afebe8097 100644 --- a/backend/open_webui/apps/retrieval/search/serper.py +++ b/backend/open_webui/apps/retrieval/search/serper.py @@ -3,7 +3,7 @@ import logging from typing import Optional import requests -from open_webui.apps.rag.search.main import SearchResult, get_filtered_results +from open_webui.apps.retrieval.search.main import SearchResult, get_filtered_results from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) diff --git a/backend/open_webui/apps/retrieval/search/serply.py b/backend/open_webui/apps/retrieval/search/serply.py index 260e9b30e..266fd666a 100644 --- a/backend/open_webui/apps/retrieval/search/serply.py +++ b/backend/open_webui/apps/retrieval/search/serply.py @@ -3,7 +3,7 @@ from typing import Optional from urllib.parse import urlencode import requests -from open_webui.apps.rag.search.main import SearchResult, get_filtered_results +from open_webui.apps.retrieval.search.main import SearchResult, get_filtered_results from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) diff --git a/backend/open_webui/apps/retrieval/search/serpstack.py b/backend/open_webui/apps/retrieval/search/serpstack.py index 962c1a5b3..236fb5181 100644 --- a/backend/open_webui/apps/retrieval/search/serpstack.py +++ b/backend/open_webui/apps/retrieval/search/serpstack.py @@ -2,7 +2,7 @@ import logging from typing import Optional import requests -from open_webui.apps.rag.search.main import SearchResult, get_filtered_results +from open_webui.apps.retrieval.search.main import SearchResult, get_filtered_results from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) diff --git a/backend/open_webui/apps/retrieval/search/tavily.py b/backend/open_webui/apps/retrieval/search/tavily.py index a619d29ed..00f5b15c4 100644 --- a/backend/open_webui/apps/retrieval/search/tavily.py +++ b/backend/open_webui/apps/retrieval/search/tavily.py @@ -1,7 +1,7 @@ import logging import requests -from open_webui.apps.rag.search.main import SearchResult +from open_webui.apps.retrieval.search.main import SearchResult from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) diff --git a/backend/open_webui/apps/retrieval/utils.py b/backend/open_webui/apps/retrieval/utils.py index f9443d380..1fa30e6a0 100644 --- a/backend/open_webui/apps/retrieval/utils.py +++ b/backend/open_webui/apps/retrieval/utils.py @@ -15,7 +15,7 @@ from open_webui.apps.ollama.main import ( GenerateEmbeddingsForm, generate_ollama_embeddings, ) -from open_webui.apps.rag.vector.connector import VECTOR_DB_CLIENT +from open_webui.apps.retrieval.vector.connector import VECTOR_DB_CLIENT from open_webui.utils.misc import get_last_user_message from open_webui.env import SRC_LOG_LEVELS diff --git a/backend/open_webui/apps/retrieval/vector/connector.py b/backend/open_webui/apps/retrieval/vector/connector.py index 073becdbe..5b203271f 100644 --- a/backend/open_webui/apps/retrieval/vector/connector.py +++ b/backend/open_webui/apps/retrieval/vector/connector.py @@ -1,5 +1,5 @@ -from open_webui.apps.rag.vector.dbs.chroma import ChromaClient -from open_webui.apps.rag.vector.dbs.milvus import MilvusClient +from open_webui.apps.retrieval.vector.dbs.chroma import ChromaClient +from open_webui.apps.retrieval.vector.dbs.milvus import MilvusClient from open_webui.config import VECTOR_DB diff --git a/backend/open_webui/apps/retrieval/vector/dbs/chroma.py b/backend/open_webui/apps/retrieval/vector/dbs/chroma.py index 5f9420108..fe065f868 100644 --- a/backend/open_webui/apps/retrieval/vector/dbs/chroma.py +++ b/backend/open_webui/apps/retrieval/vector/dbs/chroma.py @@ -4,7 +4,7 @@ from chromadb.utils.batch_utils import create_batches from typing import Optional -from open_webui.apps.rag.vector.main import VectorItem, SearchResult, GetResult +from open_webui.apps.retrieval.vector.main import VectorItem, SearchResult, GetResult from open_webui.config import ( CHROMA_DATA_PATH, CHROMA_HTTP_HOST, diff --git a/backend/open_webui/apps/retrieval/vector/dbs/milvus.py b/backend/open_webui/apps/retrieval/vector/dbs/milvus.py index b4a6a77b3..77300acf2 100644 --- a/backend/open_webui/apps/retrieval/vector/dbs/milvus.py +++ b/backend/open_webui/apps/retrieval/vector/dbs/milvus.py @@ -4,7 +4,7 @@ import json from typing import Optional -from open_webui.apps.rag.vector.main import VectorItem, SearchResult, GetResult +from open_webui.apps.retrieval.vector.main import VectorItem, SearchResult, GetResult from open_webui.config import ( MILVUS_URI, ) diff --git a/backend/open_webui/apps/webui/routers/memories.py b/backend/open_webui/apps/webui/routers/memories.py index d659833bc..ccf84a9d4 100644 --- a/backend/open_webui/apps/webui/routers/memories.py +++ b/backend/open_webui/apps/webui/routers/memories.py @@ -4,7 +4,7 @@ import logging from typing import Optional from open_webui.apps.webui.models.memories import Memories, MemoryModel -from open_webui.apps.rag.vector.connector import VECTOR_DB_CLIENT +from open_webui.apps.retrieval.vector.connector import VECTOR_DB_CLIENT from open_webui.utils.utils import get_verified_user from open_webui.env import SRC_LOG_LEVELS From a52e8cd5375e7347dc7c4bb74a6b1a114db40fe1 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 01:35:31 +0200 Subject: [PATCH 12/19] refac --- backend/open_webui/apps/retrieval/main.py | 8 ++++---- backend/open_webui/config.py | 2 +- src/lib/apis/{rag => retrieval}/index.ts | 4 ++-- src/lib/components/chat/Chat.svelte | 2 +- src/lib/components/chat/MessageInput.svelte | 5 +++-- src/lib/components/workspace/Documents.svelte | 4 ++-- src/lib/utils/rag/index.ts | 2 +- 7 files changed, 14 insertions(+), 13 deletions(-) rename src/lib/apis/{rag => retrieval}/index.ts (98%) diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index e4f98b8b0..8f23ea2c5 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -1340,14 +1340,14 @@ def store_doc( ) -class ProcessDocForm(BaseModel): +class ProcessFileForm(BaseModel): file_id: str collection_name: Optional[str] = None -@app.post("/process/doc") -def process_doc( - form_data: ProcessDocForm, +@app.post("/process/file") +def process_file( + form_data: ProcessFileForm, user=Depends(get_verified_user), ): try: diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index f531a8728..2518599ca 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -921,7 +921,7 @@ CHROMA_HTTP_SSL = os.environ.get("CHROMA_HTTP_SSL", "false").lower() == "true" MILVUS_URI = os.environ.get("MILVUS_URI", f"{DATA_DIR}/vector_db/milvus.db") #################################### -# RAG +# Information Retrieval (RAG) #################################### # RAG Content Extraction diff --git a/src/lib/apis/rag/index.ts b/src/lib/apis/retrieval/index.ts similarity index 98% rename from src/lib/apis/rag/index.ts rename to src/lib/apis/retrieval/index.ts index 3c0dba4b5..ce3a0c0a5 100644 --- a/src/lib/apis/rag/index.ts +++ b/src/lib/apis/retrieval/index.ts @@ -170,10 +170,10 @@ export const updateQuerySettings = async (token: string, settings: QuerySettings return res; }; -export const processDocToVectorDB = async (token: string, file_id: string) => { +export const processFile = async (token: string, file_id: string) => { let error = null; - const res = await fetch(`${RAG_API_BASE_URL}/process/doc`, { + const res = await fetch(`${RAG_API_BASE_URL}/process/file`, { method: 'POST', headers: { Accept: 'application/json', diff --git a/src/lib/components/chat/Chat.svelte b/src/lib/components/chat/Chat.svelte index 9db03ef53..e196936a6 100644 --- a/src/lib/components/chat/Chat.svelte +++ b/src/lib/components/chat/Chat.svelte @@ -52,7 +52,7 @@ updateChatById } from '$lib/apis/chats'; import { generateOpenAIChatCompletion } from '$lib/apis/openai'; - import { runWebSearch } from '$lib/apis/rag'; + import { runWebSearch } from '$lib/apis/retrieval'; import { createOpenAITextStream } from '$lib/apis/streaming'; import { queryMemory } from '$lib/apis/memories'; import { getAndUpdateUserLocation, getUserSettings } from '$lib/apis/users'; diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 0a10d5393..2ca0c8d20 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -17,7 +17,8 @@ import { blobToFile, findWordIndices } from '$lib/utils'; import { transcribeAudio } from '$lib/apis/audio'; - import { processDocToVectorDB } from '$lib/apis/rag'; + + import { processFile } from '$lib/apis/retrieval'; import { uploadFile } from '$lib/apis/files'; import { @@ -158,7 +159,7 @@ const processFileItem = async (fileItem) => { try { - const res = await processDocToVectorDB(localStorage.token, fileItem.id); + const res = await processFile(localStorage.token, fileItem.id); if (res) { fileItem.status = 'processed'; fileItem.collection_name = res.collection_name; diff --git a/src/lib/components/workspace/Documents.svelte b/src/lib/components/workspace/Documents.svelte index 38f46f745..bba4af10f 100644 --- a/src/lib/components/workspace/Documents.svelte +++ b/src/lib/components/workspace/Documents.svelte @@ -8,7 +8,7 @@ import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents'; import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants'; - import { processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag'; + import { processFile } from '$lib/apis/rag'; import { blobToFile, transformFileName } from '$lib/utils'; import Checkbox from '$lib/components/common/Checkbox.svelte'; @@ -74,7 +74,7 @@ return null; }); - const res = await processDocToVectorDB(localStorage.token, uploadedFile.id).catch((error) => { + const res = await processFile(localStorage.token, uploadedFile.id).catch((error) => { toast.error(error); return null; }); diff --git a/src/lib/utils/rag/index.ts b/src/lib/utils/rag/index.ts index ba1f29f88..6523bb7df 100644 --- a/src/lib/utils/rag/index.ts +++ b/src/lib/utils/rag/index.ts @@ -1,4 +1,4 @@ -import { getRAGTemplate } from '$lib/apis/rag'; +import { getRAGTemplate } from '$lib/apis/retrieval'; export const RAGTemplate = async (token: string, context: string, query: string) => { let template = await getRAGTemplate(token).catch(() => { From c1b4fbf5c2002634cd3d9e6e8e2b25f51a580425 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 01:35:52 +0200 Subject: [PATCH 13/19] refac --- src/lib/components/workspace/Documents.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/workspace/Documents.svelte b/src/lib/components/workspace/Documents.svelte index bba4af10f..0fa50278c 100644 --- a/src/lib/components/workspace/Documents.svelte +++ b/src/lib/components/workspace/Documents.svelte @@ -8,7 +8,7 @@ import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents'; import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants'; - import { processFile } from '$lib/apis/rag'; + import { processFile } from '$lib/apis/retrieval'; import { blobToFile, transformFileName } from '$lib/utils'; import Checkbox from '$lib/components/common/Checkbox.svelte'; From 1b349016ffaef65ad4e4713689d21b4752bbc815 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 01:36:35 +0200 Subject: [PATCH 14/19] refac --- src/lib/components/admin/Settings/Documents.svelte | 2 +- src/lib/components/admin/Settings/WebSearch.svelte | 2 +- src/lib/components/chat/MessageInput/Commands.svelte | 2 +- src/lib/components/documents/AddDocModal.svelte | 9 +++------ 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index e06edce9d..84f60847e 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -17,7 +17,7 @@ resetUploadDir, getRAGConfig, updateRAGConfig - } from '$lib/apis/rag'; + } from '$lib/apis/retrieval'; import ResetUploadDirConfirmDialog from '$lib/components/common/ConfirmDialog.svelte'; import ResetVectorDBConfirmDialog from '$lib/components/common/ConfirmDialog.svelte'; diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 15eba096b..0a0c2eb16 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -1,5 +1,5 @@
@@ -37,17 +35,7 @@ class="h-14 {className} flex items-center space-x-3 {colorClassName} rounded-xl border border-gray-100 dark:border-gray-800 text-left" type="button" on:click={async () => { - if (clickHandler === null) { - if (url) { - if (type === 'file') { - window.open(`${url}/content`, '_blank').focus(); - } else { - window.open(`${url}`, '_blank').focus(); - } - } - } else { - clickHandler(); - } + dispatch('click'); }} >
From 2428878f4225f644b285ee6c1c1d251db96b165a Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 02:29:08 +0200 Subject: [PATCH 16/19] refac --- backend/open_webui/apps/retrieval/main.py | 96 ++++------------------- 1 file changed, 16 insertions(+), 80 deletions(-) diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index 3e1ec8854..497a5685d 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -246,10 +246,10 @@ app.add_middleware( class CollectionNameForm(BaseModel): - collection_name: Optional[str] = "test" + collection_name: Optional[str] = None -class UrlForm(CollectionNameForm): +class ProcessUrlForm(CollectionNameForm): url: str @@ -636,7 +636,6 @@ def store_data_in_vector_db( chunk_overlap=app.state.config.CHUNK_OVERLAP, add_start_index=True, ) - docs = text_splitter.split_documents(data) if len(docs) > 0: @@ -715,66 +714,6 @@ def store_docs_in_vector_db( return False -@app.post("/doc") -def store_doc( - collection_name: Optional[str] = Form(None), - file: UploadFile = File(...), - user=Depends(get_verified_user), -): - # "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm" - - log.info(f"file.content_type: {file.content_type}") - try: - unsanitized_filename = file.filename - filename = os.path.basename(unsanitized_filename) - - file_path = f"{UPLOAD_DIR}/{filename}" - - contents = file.file.read() - with open(file_path, "wb") as f: - f.write(contents) - f.close() - - f = open(file_path, "rb") - if collection_name is None: - collection_name = calculate_sha256(f)[:63] - f.close() - - loader = Loader( - engine=app.state.config.CONTENT_EXTRACTION_ENGINE, - TIKA_SERVER_URL=app.state.config.TIKA_SERVER_URL, - PDF_EXTRACT_IMAGES=app.state.config.PDF_EXTRACT_IMAGES, - ) - data = loader.load(filename, file.content_type, file_path) - - try: - result = store_data_in_vector_db(data, collection_name) - - if result: - return { - "status": True, - "collection_name": collection_name, - "filename": filename, - } - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=e, - ) - except Exception as e: - log.exception(e) - if "No pandoc was found" in str(e): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=ERROR_MESSAGES.PANDOC_NOT_INSTALLED, - ) - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=ERROR_MESSAGES.DEFAULT(e), - ) - - class ProcessFileForm(BaseModel): file_id: str collection_name: Optional[str] = None @@ -796,11 +735,10 @@ def process_file( ) data = loader.load(file.filename, file.meta.get("content_type"), file_path) - f = open(file_path, "rb") collection_name = form_data.collection_name if collection_name is None: - collection_name = calculate_sha256(f)[:63] - f.close() + with open(file_path, "rb") as f: + collection_name = calculate_sha256(f)[:63] try: result = store_data_in_vector_db( @@ -813,11 +751,9 @@ def process_file( ) if result: - return { "status": True, "collection_name": collection_name, - "known_type": known_type, "filename": file.meta.get("name", file.filename), } except Exception as e: @@ -839,15 +775,15 @@ def process_file( ) -class TextRAGForm(BaseModel): +class ProcessTextForm(BaseModel): name: str content: str collection_name: Optional[str] = None -@app.post("/text") -def store_text( - form_data: TextRAGForm, +@app.post("/process/text") +def process_text( + form_data: ProcessTextForm, user=Depends(get_verified_user), ): collection_name = form_data.collection_name @@ -878,9 +814,8 @@ def process_docs_dir(user=Depends(get_admin_user)): filename = path.name file_content_type = mimetypes.guess_type(path) - f = open(path, "rb") - collection_name = calculate_sha256(f)[:63] - f.close() + with open(path, "rb") as f: + collection_name = calculate_sha256(f)[:63] loader = Loader( engine=app.state.config.CONTENT_EXTRACTION_ENGINE, @@ -933,7 +868,7 @@ def process_docs_dir(user=Depends(get_admin_user)): @app.post("/process/youtube") -def process_youtube_video(form_data: UrlForm, user=Depends(get_verified_user)): +def process_youtube_video(form_data: ProcessUrlForm, user=Depends(get_verified_user)): try: loader = YoutubeLoader.from_youtube_url( form_data.url, @@ -944,10 +879,11 @@ def process_youtube_video(form_data: UrlForm, user=Depends(get_verified_user)): data = loader.load() collection_name = form_data.collection_name - if collection_name == "": + if not collection_name: collection_name = calculate_sha256_string(form_data.url)[:63] store_data_in_vector_db(data, collection_name, overwrite=True) + return { "status": True, "collection_name": collection_name, @@ -962,8 +898,7 @@ def process_youtube_video(form_data: UrlForm, user=Depends(get_verified_user)): @app.post("/process/web") -def process_web(form_data: UrlForm, user=Depends(get_verified_user)): - # "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm" +def process_web(form_data: ProcessUrlForm, user=Depends(get_verified_user)): try: loader = get_web_loader( form_data.url, @@ -973,10 +908,11 @@ def process_web(form_data: UrlForm, user=Depends(get_verified_user)): data = loader.load() collection_name = form_data.collection_name - if collection_name == "": + if not collection_name: collection_name = calculate_sha256_string(form_data.url)[:63] store_data_in_vector_db(data, collection_name, overwrite=True) + return { "status": True, "collection_name": collection_name, From 00eb02245027220788d51675628978d8e75ce603 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 02:38:59 +0200 Subject: [PATCH 17/19] refac --- backend/open_webui/apps/retrieval/main.py | 119 ++++++++++------------ 1 file changed, 55 insertions(+), 64 deletions(-) diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index 497a5685d..f2f4733c5 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -628,40 +628,26 @@ async def update_query_settings( #################################### -def store_data_in_vector_db( - data, collection_name, metadata: Optional[dict] = None, overwrite: bool = False +def save_docs_to_vector_db( + docs, + collection_name, + metadata: Optional[dict] = None, + overwrite: bool = False, + split: bool = True, ) -> bool: - text_splitter = RecursiveCharacterTextSplitter( - chunk_size=app.state.config.CHUNK_SIZE, - chunk_overlap=app.state.config.CHUNK_OVERLAP, - add_start_index=True, - ) - docs = text_splitter.split_documents(data) + log.info(f"save_docs_to_vector_db {docs} {collection_name}") - if len(docs) > 0: - log.info(f"store_data_in_vector_db {docs}") - return store_docs_in_vector_db(docs, collection_name, metadata, overwrite) - else: + if split: + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=app.state.config.CHUNK_SIZE, + chunk_overlap=app.state.config.CHUNK_OVERLAP, + add_start_index=True, + ) + docs = text_splitter.split_documents(docs) + + if len(docs) == 0: raise ValueError(ERROR_MESSAGES.EMPTY_CONTENT) - -def store_text_in_vector_db( - text, metadata, collection_name, overwrite: bool = False -) -> bool: - text_splitter = RecursiveCharacterTextSplitter( - chunk_size=app.state.config.CHUNK_SIZE, - chunk_overlap=app.state.config.CHUNK_OVERLAP, - add_start_index=True, - ) - docs = text_splitter.create_documents([text], metadatas=[metadata]) - return store_docs_in_vector_db(docs, collection_name, overwrite=overwrite) - - -def store_docs_in_vector_db( - docs, collection_name, metadata: Optional[dict] = None, overwrite: bool = False -) -> bool: - log.info(f"store_docs_in_vector_db {docs} {collection_name}") - texts = [doc.page_content for doc in docs] metadatas = [{**doc.metadata, **(metadata if metadata else {})} for doc in docs] @@ -728,21 +714,24 @@ def process_file( file = Files.get_file_by_id(form_data.file_id) file_path = file.meta.get("path", f"{UPLOAD_DIR}/{file.filename}") - loader = Loader( - engine=app.state.config.CONTENT_EXTRACTION_ENGINE, - TIKA_SERVER_URL=app.state.config.TIKA_SERVER_URL, - PDF_EXTRACT_IMAGES=app.state.config.PDF_EXTRACT_IMAGES, - ) - data = loader.load(file.filename, file.meta.get("content_type"), file_path) - collection_name = form_data.collection_name if collection_name is None: with open(file_path, "rb") as f: collection_name = calculate_sha256(f)[:63] + loader = Loader( + engine=app.state.config.CONTENT_EXTRACTION_ENGINE, + TIKA_SERVER_URL=app.state.config.TIKA_SERVER_URL, + PDF_EXTRACT_IMAGES=app.state.config.PDF_EXTRACT_IMAGES, + ) + docs = loader.load(file.filename, file.meta.get("content_type"), file_path) + + raw_content = " ".join([doc.page_content for doc in docs]) + print(raw_content) + try: - result = store_data_in_vector_db( - data, + result = save_docs_to_vector_db( + docs, collection_name, { "file_id": form_data.file_id, @@ -790,11 +779,13 @@ def process_text( if collection_name is None: collection_name = calculate_sha256_string(form_data.content) - result = store_text_in_vector_db( - form_data.content, - metadata={"name": form_data.name, "created_by": user.id}, - collection_name=collection_name, - ) + docs = [ + Document( + page_content=form_data.content, + metadata={"name": form_data.name, "created_by": user.id}, + ) + ] + result = save_docs_to_vector_db(docs, collection_name) if result: return {"status": True, "collection_name": collection_name} @@ -822,10 +813,10 @@ def process_docs_dir(user=Depends(get_admin_user)): TIKA_SERVER_URL=app.state.config.TIKA_SERVER_URL, PDF_EXTRACT_IMAGES=app.state.config.PDF_EXTRACT_IMAGES, ) - data = loader.load(filename, file_content_type[0], str(path)) + docs = loader.load(filename, file_content_type[0], str(path)) try: - result = store_data_in_vector_db(data, collection_name) + result = save_docs_to_vector_db(docs, collection_name) if result: sanitized_filename = sanitize_filename(filename) @@ -870,19 +861,19 @@ def process_docs_dir(user=Depends(get_admin_user)): @app.post("/process/youtube") def process_youtube_video(form_data: ProcessUrlForm, user=Depends(get_verified_user)): try: + collection_name = form_data.collection_name + if not collection_name: + collection_name = calculate_sha256_string(form_data.url)[:63] + loader = YoutubeLoader.from_youtube_url( form_data.url, add_video_info=True, language=app.state.config.YOUTUBE_LOADER_LANGUAGE, translation=app.state.YOUTUBE_LOADER_TRANSLATION, ) - data = loader.load() + docs = loader.load() - collection_name = form_data.collection_name - if not collection_name: - collection_name = calculate_sha256_string(form_data.url)[:63] - - store_data_in_vector_db(data, collection_name, overwrite=True) + save_docs_to_vector_db(docs, collection_name, overwrite=True) return { "status": True, @@ -900,18 +891,17 @@ def process_youtube_video(form_data: ProcessUrlForm, user=Depends(get_verified_u @app.post("/process/web") def process_web(form_data: ProcessUrlForm, user=Depends(get_verified_user)): try: + collection_name = form_data.collection_name + if not collection_name: + collection_name = calculate_sha256_string(form_data.url)[:63] + loader = get_web_loader( form_data.url, verify_ssl=app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION, requests_per_second=app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS, ) - data = loader.load() - - collection_name = form_data.collection_name - if not collection_name: - collection_name = calculate_sha256_string(form_data.url)[:63] - - store_data_in_vector_db(data, collection_name, overwrite=True) + docs = loader.load() + save_docs_to_vector_db(docs, collection_name, overwrite=True) return { "status": True, @@ -1060,15 +1050,16 @@ def process_web_search(form_data: SearchForm, user=Depends(get_verified_user)): ) try: - urls = [result.link for result in web_results] - loader = get_web_loader(urls) - data = loader.load() - collection_name = form_data.collection_name if collection_name == "": collection_name = calculate_sha256_string(form_data.query)[:63] - store_data_in_vector_db(data, collection_name, overwrite=True) + urls = [result.link for result in web_results] + + loader = get_web_loader(urls) + docs = loader.load() + save_docs_to_vector_db(docs, collection_name, overwrite=True) + return { "status": True, "collection_name": collection_name, From b8b994a82040a79c37bd6e73bc11f1dd264e2c61 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 02:49:18 +0200 Subject: [PATCH 18/19] refac --- backend/open_webui/apps/retrieval/loader/main.py | 11 ++++++++++- backend/open_webui/apps/retrieval/main.py | 2 -- backend/requirements.txt | 2 ++ pyproject.toml | 2 ++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/apps/retrieval/loader/main.py b/backend/open_webui/apps/retrieval/loader/main.py index f4c948b43..b435fa21f 100644 --- a/backend/open_webui/apps/retrieval/loader/main.py +++ b/backend/open_webui/apps/retrieval/loader/main.py @@ -1,5 +1,7 @@ import requests import logging +import ftfy + from langchain_community.document_loaders import ( BSHTMLLoader, @@ -122,7 +124,14 @@ class Loader: self, filename: str, file_content_type: str, file_path: str ) -> list[Document]: loader = self._get_loader(filename, file_content_type, file_path) - return loader.load() + docs = loader.load() + + return [ + Document( + page_content=ftfy.fix_text(doc.page_content), metadata=doc.metadata + ) + for doc in docs + ] def _get_loader(self, filename: str, file_content_type: str, file_path: str): file_ext = filename.split(".")[-1].lower() diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index f2f4733c5..9c2ec141f 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -725,7 +725,6 @@ def process_file( PDF_EXTRACT_IMAGES=app.state.config.PDF_EXTRACT_IMAGES, ) docs = loader.load(file.filename, file.meta.get("content_type"), file_path) - raw_content = " ".join([doc.page_content for doc in docs]) print(raw_content) @@ -872,7 +871,6 @@ def process_youtube_video(form_data: ProcessUrlForm, user=Depends(get_verified_u translation=app.state.YOUTUBE_LOADER_TRANSLATION, ) docs = loader.load() - save_docs_to_vector_db(docs, collection_name, overwrite=True) return { diff --git a/backend/requirements.txt b/backend/requirements.txt index 764e41d3d..a6933d20a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -46,6 +46,8 @@ sentence-transformers==3.0.1 colbert-ai==0.2.21 einops==0.8.0 + +ftfy==6.2.3 pypdf==4.3.1 docx2txt==0.8 python-pptx==1.0.0 diff --git a/pyproject.toml b/pyproject.toml index d02281d52..1df284f80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,8 @@ dependencies = [ "colbert-ai==0.2.21", "einops==0.8.0", + + "ftfy==6.2.3", "pypdf==4.3.1", "docx2txt==0.8", "python-pptx==1.0.0", From 9d2ed3d2be6b94f2866c549688740b92f67f6568 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 28 Sep 2024 02:56:56 +0200 Subject: [PATCH 19/19] refac --- backend/open_webui/apps/retrieval/loader/main.py | 2 -- backend/open_webui/apps/retrieval/main.py | 12 ++++++++++-- backend/open_webui/apps/webui/models/files.py | 11 +++++++++++ backend/open_webui/apps/webui/routers/files.py | 13 +++++++++++++ 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/backend/open_webui/apps/retrieval/loader/main.py b/backend/open_webui/apps/retrieval/loader/main.py index b435fa21f..f0e8f804e 100644 --- a/backend/open_webui/apps/retrieval/loader/main.py +++ b/backend/open_webui/apps/retrieval/loader/main.py @@ -2,7 +2,6 @@ import requests import logging import ftfy - from langchain_community.document_loaders import ( BSHTMLLoader, CSVLoader, @@ -24,7 +23,6 @@ from open_webui.env import SRC_LOG_LEVELS log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["RAG"]) - known_source_ext = [ "go", "py", diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index 9c2ec141f..a3e828978 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -725,8 +725,16 @@ def process_file( PDF_EXTRACT_IMAGES=app.state.config.PDF_EXTRACT_IMAGES, ) docs = loader.load(file.filename, file.meta.get("content_type"), file_path) - raw_content = " ".join([doc.page_content for doc in docs]) - print(raw_content) + raw_text_content = " ".join([doc.page_content for doc in docs]) + + Files.update_files_metadata_by_id( + form_data.file_id, + { + "content": { + "text": raw_text_content, + } + }, + ) try: result = save_docs_to_vector_db( diff --git a/backend/open_webui/apps/webui/models/files.py b/backend/open_webui/apps/webui/models/files.py index 7fba74479..cf572ac78 100644 --- a/backend/open_webui/apps/webui/models/files.py +++ b/backend/open_webui/apps/webui/models/files.py @@ -97,6 +97,17 @@ class FilesTable: for file in db.query(File).filter_by(user_id=user_id).all() ] + def update_files_metadata_by_id(self, id: str, meta: dict) -> Optional[FileModel]: + with get_db() as db: + try: + file = db.query(File).filter_by(id=id).first() + file.meta = {**file.meta, **meta} + db.commit() + + return FileModel.model_validate(file) + except Exception: + return None + def delete_file_by_id(self, id: str) -> bool: with get_db() as db: try: diff --git a/backend/open_webui/apps/webui/routers/files.py b/backend/open_webui/apps/webui/routers/files.py index 1a326bcd8..f46a7992d 100644 --- a/backend/open_webui/apps/webui/routers/files.py +++ b/backend/open_webui/apps/webui/routers/files.py @@ -171,6 +171,19 @@ async def get_file_content_by_id(id: str, user=Depends(get_verified_user)): ) +@router.get("/{id}/content/text") +async def get_file_text_content_by_id(id: str, user=Depends(get_verified_user)): + file = Files.get_file_by_id(id) + + if file and (file.user_id == user.id or user.role == "admin"): + return {"text": file.meta.get("content", {}).get("text", None)} + else: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=ERROR_MESSAGES.NOT_FOUND, + ) + + @router.get("/{id}/content/{file_name}", response_model=Optional[FileModel]) async def get_file_content_by_id(id: str, user=Depends(get_verified_user)): file = Files.get_file_by_id(id)