From bfb12a7851d4cd3c1828661db68bbed53c15dcc8 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 6 Sep 2024 04:59:20 +0200 Subject: [PATCH 01/16] refac --- backend/.gitignore | 6 +----- backend/open_webui/__init__.py | 2 ++ backend/open_webui/env.py | 20 ++++---------------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/backend/.gitignore b/backend/.gitignore index ea83b34f4..614a5f746 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -8,9 +8,5 @@ _test Pipfile !/data /data/* -!/data/litellm -/data/litellm/* -!data/litellm/config.yaml - -!data/config.json +/open_webui/data/* .webui_secret_key \ No newline at end of file diff --git a/backend/open_webui/__init__.py b/backend/open_webui/__init__.py index c6578c214..bb1db0dea 100644 --- a/backend/open_webui/__init__.py +++ b/backend/open_webui/__init__.py @@ -10,6 +10,8 @@ app = typer.Typer() KEY_FILE = Path.cwd() / ".webui_secret_key" +os.environ["RUNNING_FROM_INIT_PY"] = "true" + @app.command() def serve( diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index b0585db90..865d61003 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -88,21 +88,9 @@ WEBUI_FAVICON_URL = "https://openwebui.com/favicon.png" ENV = os.environ.get("ENV", "dev") -PIP_INSTALL = False -try: - importlib.metadata.version("open-webui") - PIP_INSTALL = True -except importlib.metadata.PackageNotFoundError: - pass +FROM_INIT_PY = os.environ.get("FROM_INIT_PY", "False").lower() == "true" - -PIP_INSTALL = ( - os.environ.get("PIP_INSTALL", "False").lower() == "true" - if os.environ.get("PIP_INSTALL") - else PIP_INSTALL -) - -if PIP_INSTALL: +if FROM_INIT_PY: PACKAGE_DATA = {"version": importlib.metadata.version("open-webui")} else: try: @@ -193,7 +181,7 @@ WEBUI_BUILD_HASH = os.environ.get("WEBUI_BUILD_HASH", "dev-build") DATA_DIR = Path(os.getenv("DATA_DIR", BACKEND_DIR / "data")).resolve() -if PIP_INSTALL: +if FROM_INIT_PY: NEW_DATA_DIR = Path(os.getenv("DATA_DIR", OPEN_WEBUI_DIR / "data")).resolve() NEW_DATA_DIR.mkdir(parents=True, exist_ok=True) @@ -212,7 +200,7 @@ if PIP_INSTALL: FRONTEND_BUILD_DIR = Path(os.getenv("FRONTEND_BUILD_DIR", BASE_DIR / "build")).resolve() -if PIP_INSTALL: +if FROM_INIT_PY: FRONTEND_BUILD_DIR = Path( os.getenv("FRONTEND_BUILD_DIR", OPEN_WEBUI_DIR / "frontend") ).resolve() From 9fe62fc80d2a9025ffd91fd7410e6497144a1bb3 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 6 Sep 2024 05:30:16 +0200 Subject: [PATCH 02/16] refac --- .../open_webui/migrations/scripts/revision.py | 19 +++++++++ .../migrations/versions/7e5b5dc7342b_init.py | 42 +++++-------------- 2 files changed, 30 insertions(+), 31 deletions(-) create mode 100644 backend/open_webui/migrations/scripts/revision.py diff --git a/backend/open_webui/migrations/scripts/revision.py b/backend/open_webui/migrations/scripts/revision.py new file mode 100644 index 000000000..32ebc9e35 --- /dev/null +++ b/backend/open_webui/migrations/scripts/revision.py @@ -0,0 +1,19 @@ +from alembic import command +from alembic.config import Config + +from open_webui.env import OPEN_WEBUI_DIR + +alembic_cfg = Config(OPEN_WEBUI_DIR / "alembic.ini") + +# Set the script location dynamically +migrations_path = OPEN_WEBUI_DIR / "migrations" +alembic_cfg.set_main_option("script_location", str(migrations_path)) + + +def revision(message: str) -> None: + command.revision(alembic_cfg, message=message, autogenerate=False) + + +if __name__ == "__main__": + input_message = input("Enter the revision message: ") + revision(input_message) diff --git a/backend/open_webui/migrations/versions/7e5b5dc7342b_init.py b/backend/open_webui/migrations/versions/7e5b5dc7342b_init.py index 62d0f8588..607a7b2c9 100644 --- a/backend/open_webui/migrations/versions/7e5b5dc7342b_init.py +++ b/backend/open_webui/migrations/versions/7e5b5dc7342b_init.py @@ -11,8 +11,8 @@ from typing import Sequence, Union import sqlalchemy as sa from alembic import op - import open_webui.apps.webui.internal.db +from open_webui.apps.webui.internal.db import JSONField from open_webui.migrations.util import get_existing_tables # revision identifiers, used by Alembic. @@ -82,9 +82,7 @@ def upgrade() -> None: sa.Column("id", sa.String(), nullable=False), sa.Column("user_id", sa.String(), nullable=True), sa.Column("filename", sa.Text(), nullable=True), - sa.Column( - "meta", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), + sa.Column("meta", JSONField(), nullable=True), sa.Column("created_at", sa.BigInteger(), nullable=True), sa.PrimaryKeyConstraint("id"), ) @@ -97,12 +95,8 @@ def upgrade() -> None: sa.Column("name", sa.Text(), nullable=True), sa.Column("type", sa.Text(), nullable=True), sa.Column("content", sa.Text(), nullable=True), - sa.Column( - "meta", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), - sa.Column( - "valves", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), + sa.Column("meta", JSONField(), nullable=True), + sa.Column("valves", JSONField(), nullable=True), sa.Column("is_active", sa.Boolean(), nullable=True), sa.Column("is_global", sa.Boolean(), nullable=True), sa.Column("updated_at", sa.BigInteger(), nullable=True), @@ -128,12 +122,8 @@ def upgrade() -> None: sa.Column("user_id", sa.Text(), nullable=True), sa.Column("base_model_id", sa.Text(), nullable=True), sa.Column("name", sa.Text(), nullable=True), - sa.Column( - "params", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), - sa.Column( - "meta", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), + sa.Column("params", JSONField(), nullable=True), + sa.Column("meta", JSONField(), nullable=True), sa.Column("updated_at", sa.BigInteger(), nullable=True), sa.Column("created_at", sa.BigInteger(), nullable=True), sa.PrimaryKeyConstraint("id"), @@ -167,15 +157,9 @@ def upgrade() -> None: sa.Column("user_id", sa.String(), nullable=True), sa.Column("name", sa.Text(), nullable=True), sa.Column("content", sa.Text(), nullable=True), - sa.Column( - "specs", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), - sa.Column( - "meta", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), - sa.Column( - "valves", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), + sa.Column("specs", JSONField(), nullable=True), + sa.Column("meta", JSONField(), nullable=True), + sa.Column("valves", JSONField(), nullable=True), sa.Column("updated_at", sa.BigInteger(), nullable=True), sa.Column("created_at", sa.BigInteger(), nullable=True), sa.PrimaryKeyConstraint("id"), @@ -193,12 +177,8 @@ def upgrade() -> None: sa.Column("updated_at", sa.BigInteger(), nullable=True), sa.Column("created_at", sa.BigInteger(), nullable=True), sa.Column("api_key", sa.String(), nullable=True), - sa.Column( - "settings", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), - sa.Column( - "info", open_webui.apps.webui.internal.db.JSONField(), nullable=True - ), + sa.Column("settings", JSONField(), nullable=True), + sa.Column("info", JSONField(), nullable=True), sa.Column("oauth_sub", sa.Text(), nullable=True), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("api_key"), From d038e831dcf9127f934a849b34c58265804c7fd5 Mon Sep 17 00:00:00 2001 From: Aleix Dorca Date: Fri, 6 Sep 2024 07:20:21 +0200 Subject: [PATCH 03/16] i18n: Update Catalan translation.json --- src/lib/i18n/locales/ca-ES/translation.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/i18n/locales/ca-ES/translation.json b/src/lib/i18n/locales/ca-ES/translation.json index 30c1c1c1d..6117999bb 100644 --- a/src/lib/i18n/locales/ca-ES/translation.json +++ b/src/lib/i18n/locales/ca-ES/translation.json @@ -248,8 +248,8 @@ "Enter model tag (e.g. {{modelTag}})": "Introdueix l'etiqueta del model (p. ex. {{modelTag}})", "Enter Number of Steps (e.g. 50)": "Introdueix el nombre de passos (p. ex. 50)", "Enter Score": "Introdueix la puntuació", - "Enter SearchApi API Key": "", - "Enter SearchApi Engine": "", + "Enter SearchApi API Key": "Introdueix la clau API SearchApi", + "Enter SearchApi Engine": "Introdueix el motor SearchApi", "Enter Searxng Query URL": "Introdueix l'URL de consulta de Searxng", "Enter Serper API Key": "Introdueix la clau API Serper", "Enter Serply API Key": "Introdueix la clau API Serply", @@ -272,7 +272,7 @@ "Export All Chats (All Users)": "Exportar tots els xats (Tots els usuaris)", "Export chat (.json)": "Exportar el xat (.json)", "Export Chats": "Exportar els xats", - "Export Config to JSON File": "", + "Export Config to JSON File": "Exportar la configuració a un arxiu JSON", "Export Documents Mapping": "Exportar el mapatge de documents", "Export Functions": "Exportar funcions", "Export LiteLLM config.yaml": "Exportar la configuració LiteLLM config.yaml", @@ -337,7 +337,7 @@ "Image Settings": "Preferències d'imatges", "Images": "Imatges", "Import Chats": "Importar xats", - "Import Config from JSON File": "", + "Import Config from JSON File": "Importar la configuració des d'un arxiu JSON", "Import Documents Mapping": "Importar el mapatge de documents", "Import Functions": "Importar funcions", "Import Models": "Importar models", @@ -541,8 +541,8 @@ "Search Query Generation Prompt Length Threshold": "Mida màxima de la indicació de cerca de generació de consultes", "Search Result Count": "Recompte de resultats de cerca", "Search Tools": "Cercar eines", - "SearchApi API Key": "", - "SearchApi Engine": "", + "SearchApi API Key": "Clau API de SearchApi", + "SearchApi Engine": "Motor de SearchApi", "Searched {{count}} sites_one": "S'ha cercat {{count}} una pàgina", "Searched {{count}} sites_many": "S'han cercat {{count}} pàgines", "Searched {{count}} sites_other": "S'han cercat {{count}} pàgines", From 14eda1bf5bd7c4aaa42eef8b7493cf9301472b05 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 6 Sep 2024 15:52:23 +0200 Subject: [PATCH 04/16] fix: pdf download FONTS_DIR issue --- backend/open_webui/apps/webui/routers/utils.py | 7 ++----- backend/open_webui/env.py | 2 ++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/open_webui/apps/webui/routers/utils.py b/backend/open_webui/apps/webui/routers/utils.py index 731f98784..bb556b318 100644 --- a/backend/open_webui/apps/webui/routers/utils.py +++ b/backend/open_webui/apps/webui/routers/utils.py @@ -4,6 +4,7 @@ from pathlib import Path import black import markdown from open_webui.config import DATA_DIR, ENABLE_ADMIN_EXPORT +from open_webui.env import FONTS_DIR from open_webui.constants import ERROR_MESSAGES from fastapi import APIRouter, Depends, HTTPException, Response, status from fpdf import FPDF @@ -57,14 +58,10 @@ class ChatForm(BaseModel): async def download_chat_as_pdf( form_data: ChatForm, ): + global FONTS_DIR pdf = FPDF() pdf.add_page() - # When running in docker, workdir is /app/backend, so fonts is in /app/backend/static/fonts - FONTS_DIR = Path("./static/fonts") - - # Non Docker Installation - # When running using `pip install` the static directory is in the site packages. if not FONTS_DIR.exists(): FONTS_DIR = Path(site.getsitepackages()[0]) / "static/fonts" diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 865d61003..b716769c2 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -198,6 +198,8 @@ if FROM_INIT_PY: DATA_DIR = Path(os.getenv("DATA_DIR", OPEN_WEBUI_DIR / "data")) +FONTS_DIR = Path(os.getenv("FONTS_DIR", OPEN_WEBUI_DIR / "static" / "fonts")) + FRONTEND_BUILD_DIR = Path(os.getenv("FRONTEND_BUILD_DIR", BASE_DIR / "build")).resolve() if FROM_INIT_PY: From 02d5bca44d49e195374e28b9299d7444ec347268 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 6 Sep 2024 18:35:43 +0200 Subject: [PATCH 05/16] fix: tools & function not installing requirements --- backend/open_webui/apps/webui/utils.py | 29 ++++++++++---------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/backend/open_webui/apps/webui/utils.py b/backend/open_webui/apps/webui/utils.py index 2b017bf43..2d537af51 100644 --- a/backend/open_webui/apps/webui/utils.py +++ b/backend/open_webui/apps/webui/utils.py @@ -65,6 +65,7 @@ def replace_imports(content): def load_toolkit_module_by_id(toolkit_id, content=None): + if content is None: tool = Tools.get_tool_by_id(toolkit_id) if not tool: @@ -74,6 +75,10 @@ def load_toolkit_module_by_id(toolkit_id, content=None): content = replace_imports(content) Tools.update_tool_by_id(toolkit_id, {"content": content}) + else: + frontmatter = extract_frontmatter(content) + # Install required packages found within the frontmatter + install_frontmatter_requirements(frontmatter.get("requirements", "")) module_name = f"tool_{toolkit_id}" module = types.ModuleType(module_name) @@ -82,16 +87,9 @@ def load_toolkit_module_by_id(toolkit_id, content=None): try: # Executing the modified content in the created module's namespace exec(content, module.__dict__) - - # Extract frontmatter, assuming content can be treated directly as a string - frontmatter = extract_frontmatter( - content - ) # Ensure this method is adaptable to handle content strings - - # Install required packages found within the frontmatter - install_frontmatter_requirements(frontmatter.get("requirements", "")) - + frontmatter = extract_frontmatter(content) print(f"Loaded module: {module.__name__}") + # Create and return the object if the class 'Tools' is found in the module if hasattr(module, "Tools"): return module.Tools(), frontmatter @@ -112,6 +110,9 @@ def load_function_module_by_id(function_id, content=None): content = replace_imports(content) Functions.update_function_by_id(function_id, {"content": content}) + else: + frontmatter = extract_frontmatter(content) + install_frontmatter_requirements(frontmatter.get("requirements", "")) module_name = f"function_{function_id}" module = types.ModuleType(module_name) @@ -120,15 +121,7 @@ def load_function_module_by_id(function_id, content=None): try: # Execute the modified content in the created module's namespace exec(content, module.__dict__) - - # Extract the frontmatter from the content, simulate file-like behaviour - frontmatter = extract_frontmatter( - content - ) # This function needs to handle string inputs - - # Install necessary requirements specified in frontmatter - install_frontmatter_requirements(frontmatter.get("requirements", "")) - + frontmatter = extract_frontmatter(content) print(f"Loaded module: {module.__name__}") # Create appropriate object based on available class type in the module From 2f841f9f5ade466433a085da08d19cc13cf6a0c3 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 6 Sep 2024 23:40:35 +0200 Subject: [PATCH 06/16] fix: enable inline link image rendering --- .../chat/Messages/Markdown/MarkdownInlineTokens.svelte | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/components/chat/Messages/Markdown/MarkdownInlineTokens.svelte b/src/lib/components/chat/Messages/Markdown/MarkdownInlineTokens.svelte index e85848840..d192194be 100644 --- a/src/lib/components/chat/Messages/Markdown/MarkdownInlineTokens.svelte +++ b/src/lib/components/chat/Messages/Markdown/MarkdownInlineTokens.svelte @@ -30,7 +30,13 @@ {token.text} {/if} {:else if token.type === 'link'} - {token.text} + {#if token.tokens} + + + + {:else} + {token.text} + {/if} {:else if token.type === 'image'} {token.text} {:else if token.type === 'strong'} From f533c9750be2f2a4bb2795b16e41e12d8b95348e Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 7 Sep 2024 01:12:43 +0100 Subject: [PATCH 07/16] fix: close chat controls after call ends --- src/lib/components/chat/ChatControls.svelte | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/components/chat/ChatControls.svelte b/src/lib/components/chat/ChatControls.svelte index 026186a66..4bd46bd17 100644 --- a/src/lib/components/chat/ChatControls.svelte +++ b/src/lib/components/chat/ChatControls.svelte @@ -84,7 +84,17 @@ class="w-full h-full px-5 py-4 bg-white dark:shadow-lg dark:bg-gray-850 border border-gray-50 dark:border-gray-800 rounded-xl z-50 pointer-events-auto overflow-y-auto scrollbar-hidden" > {#if $showCallOverlay} - + { + show = false; + }} + /> {:else} { From 8dfbdbd8835c7760a5e4a6138c511cb550c1aba4 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 7 Sep 2024 01:28:07 +0100 Subject: [PATCH 08/16] fix: mic kept alive after call issue --- src/lib/components/chat/MessageInput/CallOverlay.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/components/chat/MessageInput/CallOverlay.svelte b/src/lib/components/chat/MessageInput/CallOverlay.svelte index c8ec20db7..320efa922 100644 --- a/src/lib/components/chat/MessageInput/CallOverlay.svelte +++ b/src/lib/components/chat/MessageInput/CallOverlay.svelte @@ -922,8 +922,8 @@