From 4752df9bd8997b9dbd3f1ca88910b382514f6206 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 1 Oct 2024 09:40:08 -0700 Subject: [PATCH 01/45] refac --- backend/open_webui/constants.py | 2 +- .../open_webui/migrations/scripts/revision.py | 19 ------------------- backend/open_webui/migrations/util.py | 5 +++++ 3 files changed, 6 insertions(+), 20 deletions(-) delete mode 100644 backend/open_webui/migrations/scripts/revision.py diff --git a/backend/open_webui/constants.py b/backend/open_webui/constants.py index 98dbe32b2..df6f9b37b 100644 --- a/backend/open_webui/constants.py +++ b/backend/open_webui/constants.py @@ -34,8 +34,8 @@ class ERROR_MESSAGES(str, Enum): ID_TAKEN = "Uh-oh! This id is already registered. Please choose another id string." MODEL_ID_TAKEN = "Uh-oh! This model id is already registered. Please choose another model id string." - NAME_TAG_TAKEN = "Uh-oh! This name tag is already registered. Please choose another name tag string." + INVALID_TOKEN = ( "Your session has expired or the token is invalid. Please sign in again." ) diff --git a/backend/open_webui/migrations/scripts/revision.py b/backend/open_webui/migrations/scripts/revision.py deleted file mode 100644 index 32ebc9e35..000000000 --- a/backend/open_webui/migrations/scripts/revision.py +++ /dev/null @@ -1,19 +0,0 @@ -from alembic import command -from alembic.config import Config - -from open_webui.env import OPEN_WEBUI_DIR - -alembic_cfg = Config(OPEN_WEBUI_DIR / "alembic.ini") - -# Set the script location dynamically -migrations_path = OPEN_WEBUI_DIR / "migrations" -alembic_cfg.set_main_option("script_location", str(migrations_path)) - - -def revision(message: str) -> None: - command.revision(alembic_cfg, message=message, autogenerate=False) - - -if __name__ == "__main__": - input_message = input("Enter the revision message: ") - revision(input_message) diff --git a/backend/open_webui/migrations/util.py b/backend/open_webui/migrations/util.py index 401bb94d0..d4cc00a68 100644 --- a/backend/open_webui/migrations/util.py +++ b/backend/open_webui/migrations/util.py @@ -1,5 +1,6 @@ from alembic import op from sqlalchemy import Inspector +import uuid def get_existing_tables(): @@ -7,3 +8,7 @@ def get_existing_tables(): inspector = Inspector.from_engine(con) tables = set(inspector.get_table_names()) return tables + + +def get_revision_id(): + return str(uuid.uuid4()).replace("-", "")[:12] From 5c9dd25459b9176002d4b5a6768c7604fcf3ecdb Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 1 Oct 2024 11:01:26 -0700 Subject: [PATCH 02/45] refac: files migration --- backend/open_webui/apps/webui/models/files.py | 25 ++++++++++++--- .../c0fbf31ca0db_update_file_table.py | 32 +++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 backend/open_webui/migrations/versions/c0fbf31ca0db_update_file_table.py diff --git a/backend/open_webui/apps/webui/models/files.py b/backend/open_webui/apps/webui/models/files.py index cf572ac78..f1262d5dd 100644 --- a/backend/open_webui/apps/webui/models/files.py +++ b/backend/open_webui/apps/webui/models/files.py @@ -5,7 +5,7 @@ from typing import Optional from open_webui.apps.webui.internal.db import Base, JSONField, get_db from open_webui.env import SRC_LOG_LEVELS from pydantic import BaseModel, ConfigDict -from sqlalchemy import BigInteger, Column, String, Text +from sqlalchemy import BigInteger, Column, String, Text, JSON log = logging.getLogger(__name__) log.setLevel(SRC_LOG_LEVELS["MODELS"]) @@ -20,19 +20,29 @@ class File(Base): id = Column(String, primary_key=True) user_id = Column(String) + hash = Column(String) + filename = Column(Text) + data = Column(JSON) meta = Column(JSONField) + created_at = Column(BigInteger) + updated_at = Column(BigInteger) class FileModel(BaseModel): + model_config = ConfigDict(from_attributes=True) + id: str user_id: str - filename: str - meta: dict - created_at: int # timestamp in epoch + hash: str - model_config = ConfigDict(from_attributes=True) + filename: str + data: dict + meta: dict + + created_at: int # timestamp in epoch + updated_at: int # timestamp in epoch #################### @@ -43,9 +53,14 @@ class FileModel(BaseModel): class FileModelResponse(BaseModel): id: str user_id: str + hash: str + filename: str + data: dict meta: dict + created_at: int # timestamp in epoch + updated_at: int # timestamp in epoch class FileForm(BaseModel): diff --git a/backend/open_webui/migrations/versions/c0fbf31ca0db_update_file_table.py b/backend/open_webui/migrations/versions/c0fbf31ca0db_update_file_table.py new file mode 100644 index 000000000..6a1f17042 --- /dev/null +++ b/backend/open_webui/migrations/versions/c0fbf31ca0db_update_file_table.py @@ -0,0 +1,32 @@ +"""Update file table + +Revision ID: c0fbf31ca0db +Revises: ca81bd47c050 +Create Date: 2024-09-20 15:26:35.241684 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c0fbf31ca0db" +down_revision: Union[str, None] = "ca81bd47c050" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("file", sa.Column("hash", sa.String(), nullable=True)) + op.add_column("file", sa.Column("data", sa.JSON(), nullable=True)) + op.add_column("file", sa.Column("updated_at", sa.BigInteger(), nullable=True)) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("file", "updated_at") + op.drop_column("file", "data") + op.drop_column("file", "hash") From a0fb4a9b848f0100e72f825353fbccb14550dcfa Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 1 Oct 2024 13:13:39 -0700 Subject: [PATCH 03/45] refac --- backend/open_webui/apps/retrieval/main.py | 8 ++------ backend/open_webui/apps/webui/models/files.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index 2a79ac90f..4d280193f 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -729,13 +729,9 @@ def process_file( text_content = " ".join([doc.page_content for doc in docs]) log.debug(f"text_content: {text_content}") - Files.update_files_metadata_by_id( + Files.update_files_data_by_id( form_data.file_id, - { - "content": { - "text": text_content, - } - }, + {"content": text_content}, ) try: diff --git a/backend/open_webui/apps/webui/models/files.py b/backend/open_webui/apps/webui/models/files.py index f1262d5dd..dcb98c20b 100644 --- a/backend/open_webui/apps/webui/models/files.py +++ b/backend/open_webui/apps/webui/models/files.py @@ -112,6 +112,17 @@ class FilesTable: for file in db.query(File).filter_by(user_id=user_id).all() ] + def update_files_data_by_id(self, id: str, data: dict) -> Optional[FileModel]: + with get_db() as db: + try: + file = db.query(File).filter_by(id=id).first() + file.data = {**file.data, **data} + db.commit() + + return FileModel.model_validate(file) + except Exception: + return None + def update_files_metadata_by_id(self, id: str, meta: dict) -> Optional[FileModel]: with get_db() as db: try: From fb083237cd145d6ceeef4cea4d85dbabad4e116b Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 1 Oct 2024 14:00:19 -0700 Subject: [PATCH 04/45] refac --- backend/open_webui/apps/webui/models/files.py | 1 + backend/open_webui/apps/webui/routers/files.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/apps/webui/models/files.py b/backend/open_webui/apps/webui/models/files.py index dcb98c20b..69abf6f1a 100644 --- a/backend/open_webui/apps/webui/models/files.py +++ b/backend/open_webui/apps/webui/models/files.py @@ -77,6 +77,7 @@ class FilesTable: **form_data.model_dump(), "user_id": user_id, "created_at": int(time.time()), + "updated_at": int(time.time()), } ) diff --git a/backend/open_webui/apps/webui/routers/files.py b/backend/open_webui/apps/webui/routers/files.py index f46a7992d..0a0e66c64 100644 --- a/backend/open_webui/apps/webui/routers/files.py +++ b/backend/open_webui/apps/webui/routers/files.py @@ -176,7 +176,7 @@ async def get_file_text_content_by_id(id: str, user=Depends(get_verified_user)): file = Files.get_file_by_id(id) if file and (file.user_id == user.id or user.role == "admin"): - return {"text": file.meta.get("content", {}).get("text", None)} + return {"text": file.data.get("content")} else: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, From bf57dd808e7e4be729eca565fb201595be95aeab Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 1 Oct 2024 17:35:10 -0700 Subject: [PATCH 05/45] feat: project migration --- backend/open_webui/config.py | 3 - backend/open_webui/migrations/util.py | 3 +- .../6a39f3d8e55c_add_project_table.py | 74 +++++++++++++++++++ 3 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 backend/open_webui/migrations/versions/6a39f3d8e55c_add_project_table.py diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 3c28ab01f..bd02c917c 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -56,9 +56,6 @@ def run_migrations(): print(f"Error: {e}") -run_migrations() - - class Config(Base): __tablename__ = "config" diff --git a/backend/open_webui/migrations/util.py b/backend/open_webui/migrations/util.py index d4cc00a68..955066602 100644 --- a/backend/open_webui/migrations/util.py +++ b/backend/open_webui/migrations/util.py @@ -1,6 +1,5 @@ from alembic import op from sqlalchemy import Inspector -import uuid def get_existing_tables(): @@ -11,4 +10,6 @@ def get_existing_tables(): def get_revision_id(): + import uuid + return str(uuid.uuid4()).replace("-", "")[:12] diff --git a/backend/open_webui/migrations/versions/6a39f3d8e55c_add_project_table.py b/backend/open_webui/migrations/versions/6a39f3d8e55c_add_project_table.py new file mode 100644 index 000000000..c6c42f64b --- /dev/null +++ b/backend/open_webui/migrations/versions/6a39f3d8e55c_add_project_table.py @@ -0,0 +1,74 @@ +"""Add project table + +Revision ID: 6a39f3d8e55c +Revises: c0fbf31ca0db +Create Date: 2024-10-01 14:02:35.241684 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import table, column, select + +revision = "6a39f3d8e55c" +down_revision = "c0fbf31ca0db" +branch_labels = None +depends_on = None + + +def upgrade(): + # Creating the 'project' table + print("Creating project table") + project_table = op.create_table( + "project", + sa.Column("id", sa.Text(), primary_key=True), + sa.Column("user_id", sa.Text(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("data", sa.JSON(), nullable=True), + sa.Column("meta", sa.JSON(), nullable=True), + sa.Column("created_at", sa.BigInteger(), nullable=False), + sa.Column("updated_at", sa.BigInteger(), nullable=True), + ) + + print("Migrating data from document table to project table") + # Representation of the existing 'document' table + document_table = table( + "document", + column("collection_name", sa.String()), + column("user_id", sa.String()), + column("name", sa.String()), + column("title", sa.Text()), + column("timestamp", sa.BigInteger()), + ) + + # Select all from existing document table + documents = op.get_bind().execute( + select( + document_table.c.collection_name, + document_table.c.user_id, + document_table.c.name, + document_table.c.title, + document_table.c.timestamp, + ) + ) + + # Insert data into project table from document table + for doc in documents: + op.get_bind().execute( + project_table.insert().values( + id=doc.collection_name, + user_id=doc.user_id, + description=doc.name, + meta={ + "legacy": True, + }, + name=doc.title, + created_at=doc.timestamp, + updated_at=doc.timestamp, # using created_at for both created_at and updated_at in project + ) + ) + + +def downgrade(): + op.drop_table("project") From c5eb0a973243d5f4e757e784b620d308364532c1 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 1 Oct 2024 17:35:35 -0700 Subject: [PATCH 06/45] refac: documents -> projects --- backend/open_webui/apps/webui/main.py | 4 +- .../open_webui/apps/webui/models/projects.py | 142 +++++++++++++++ .../open_webui/apps/webui/routers/projects.py | 95 ++++++++++ src/lib/apis/projects/index.ts | 167 +++++++++++++++++ .../admin/Settings/Documents.svelte | 14 +- .../chat/MessageInput/Commands.svelte | 4 +- .../{Documents.svelte => Projects.svelte} | 117 +++++------- .../Models/Knowledge/Selector.svelte | 21 +-- src/lib/components/workspace/Projects.svelte | 168 ++++++++++++++++++ .../workspace/Projects/CreateProject.svelte | 0 .../workspace/Projects/EditProject.svelte | 0 .../workspace/Projects/ProjectMenu.svelte | 65 +++++++ src/lib/stores/index.ts | 2 +- src/routes/(app)/+layout.svelte | 56 +++--- src/routes/(app)/workspace/+layout.svelte | 8 +- .../(app)/workspace/documents/+page.svelte | 5 - .../(app)/workspace/projects/+page.svelte | 5 + .../workspace/projects/create/+page.svelte | 5 + .../workspace/projects/edit/+page.svelte | 5 + 19 files changed, 748 insertions(+), 135 deletions(-) create mode 100644 backend/open_webui/apps/webui/models/projects.py create mode 100644 backend/open_webui/apps/webui/routers/projects.py create mode 100644 src/lib/apis/projects/index.ts rename src/lib/components/chat/MessageInput/Commands/{Documents.svelte => Projects.svelte} (64%) create mode 100644 src/lib/components/workspace/Projects.svelte create mode 100644 src/lib/components/workspace/Projects/CreateProject.svelte create mode 100644 src/lib/components/workspace/Projects/EditProject.svelte create mode 100644 src/lib/components/workspace/Projects/ProjectMenu.svelte delete mode 100644 src/routes/(app)/workspace/documents/+page.svelte create mode 100644 src/routes/(app)/workspace/projects/+page.svelte create mode 100644 src/routes/(app)/workspace/projects/create/+page.svelte create mode 100644 src/routes/(app)/workspace/projects/edit/+page.svelte diff --git a/backend/open_webui/apps/webui/main.py b/backend/open_webui/apps/webui/main.py index 6c6f197dd..e58f83654 100644 --- a/backend/open_webui/apps/webui/main.py +++ b/backend/open_webui/apps/webui/main.py @@ -10,7 +10,7 @@ from open_webui.apps.webui.routers import ( auths, chats, configs, - documents, + projects, files, functions, memories, @@ -111,7 +111,7 @@ app.include_router(auths.router, prefix="/auths", tags=["auths"]) app.include_router(users.router, prefix="/users", tags=["users"]) app.include_router(chats.router, prefix="/chats", tags=["chats"]) -app.include_router(documents.router, prefix="/documents", tags=["documents"]) +app.include_router(projects.router, prefix="/projects", tags=["projects"]) app.include_router(models.router, prefix="/models", tags=["models"]) app.include_router(prompts.router, prefix="/prompts", tags=["prompts"]) diff --git a/backend/open_webui/apps/webui/models/projects.py b/backend/open_webui/apps/webui/models/projects.py new file mode 100644 index 000000000..5b9f07090 --- /dev/null +++ b/backend/open_webui/apps/webui/models/projects.py @@ -0,0 +1,142 @@ +import json +import logging +import time +from typing import Optional + +from open_webui.apps.webui.internal.db import Base, get_db +from open_webui.env import SRC_LOG_LEVELS +from pydantic import BaseModel, ConfigDict +from sqlalchemy import BigInteger, Column, String, Text, JSON + + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["MODELS"]) + +#################### +# Projects DB Schema +#################### + + +class Project(Base): + __tablename__ = "project" + + id = Column(Text, unique=True, primary_key=True) + user_id = Column(Text) + + name = Column(Text) + description = Column(Text) + + data = Column(JSON, nullable=True) + meta = Column(JSON, nullable=True) + + created_at = Column(BigInteger) + updated_at = Column(BigInteger) + + +class ProjectModel(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: str + user_id: str + + name: str + description: str + + data: Optional[dict] = None + meta: Optional[dict] = None + + created_at: int # timestamp in epoch + updated_at: int # timestamp in epoch + + +#################### +# Forms +#################### + + +class ProjectResponse(BaseModel): + id: str + name: str + description: str + data: Optional[dict] = None + meta: Optional[dict] = None + created_at: int # timestamp in epoch + updated_at: int # timestamp in epoch + + +class ProjectForm(BaseModel): + id: str + name: str + description: str + data: Optional[dict] = None + + +class ProjectTable: + def insert_new_project( + self, user_id: str, form_data: ProjectForm + ) -> Optional[ProjectModel]: + with get_db() as db: + project = ProjectModel( + **{ + **form_data.model_dump(), + "user_id": user_id, + "created_at": int(time.time()), + "updated_at": int(time.time()), + } + ) + + try: + result = Project(**project.model_dump()) + db.add(result) + db.commit() + db.refresh(result) + if result: + return ProjectModel.model_validate(result) + else: + return None + except Exception: + return None + + def get_projects(self) -> list[ProjectModel]: + with get_db() as db: + return [ + ProjectModel.model_validate(project) + for project in db.query(Project).all() + ] + + def get_project_by_id(self, id: str) -> Optional[ProjectModel]: + try: + with get_db() as db: + project = db.query(Project).filter_by(id=id).first() + return ProjectModel.model_validate(project) if project else None + except Exception: + return None + + def update_project_by_id( + self, id: str, form_data: ProjectForm + ) -> Optional[ProjectModel]: + try: + with get_db() as db: + db.query(Project).filter_by(id=id).update( + { + "name": form_data.name, + "updated_id": int(time.time()), + } + ) + db.commit() + return self.get_project_by_id(id=form_data.id) + except Exception as e: + log.exception(e) + return None + + def delete_project_by_id(self, id: str) -> bool: + try: + with get_db() as db: + db.query(Project).filter_by(id=id).delete() + db.commit() + return True + except Exception: + return False + + +Projects = ProjectTable() diff --git a/backend/open_webui/apps/webui/routers/projects.py b/backend/open_webui/apps/webui/routers/projects.py new file mode 100644 index 000000000..ed47b41b2 --- /dev/null +++ b/backend/open_webui/apps/webui/routers/projects.py @@ -0,0 +1,95 @@ +import json +from typing import Optional, Union +from pydantic import BaseModel +from fastapi import APIRouter, Depends, HTTPException, status + + +from open_webui.apps.webui.models.projects import ( + Projects, + ProjectModel, + ProjectForm, + ProjectResponse, +) +from open_webui.constants import ERROR_MESSAGES +from open_webui.utils.utils import get_admin_user, get_verified_user + +router = APIRouter() + +############################ +# GetProjects +############################ + + +@router.get("/", response_model=Optional[Union[list[ProjectResponse], ProjectResponse]]) +async def get_projects(id: Optional[str] = None, user=Depends(get_verified_user)): + if id: + project = Projects.get_project_by_id(id=id) + + if project: + return project + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=ERROR_MESSAGES.NOT_FOUND, + ) + else: + return [ + ProjectResponse(**project.model_dump()) + for project in Projects.get_projects() + ] + + +############################ +# CreateNewProject +############################ + + +@router.post("/create", response_model=Optional[ProjectResponse]) +async def create_new_project(form_data: ProjectForm, user=Depends(get_admin_user)): + project = Projects.get_project_by_id(form_data.id) + if project is None: + project = Projects.insert_new_project(user.id, form_data) + + if project: + return project + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.FILE_EXISTS, + ) + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.ID_TAKEN, + ) + + +############################ +# UpdateProjectById +############################ + + +@router.post("/update", response_model=Optional[ProjectResponse]) +async def update_project_by_id( + form_data: ProjectForm, + user=Depends(get_admin_user), +): + project = Projects.update_project_by_id(form_data) + if project: + return project + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.ID_TAKEN, + ) + + +############################ +# DeleteProjectById +############################ + + +@router.delete("/delete", response_model=bool) +async def delete_project_by_id(id: str, user=Depends(get_admin_user)): + result = Projects.delete_project_by_id(id=id) + return result diff --git a/src/lib/apis/projects/index.ts b/src/lib/apis/projects/index.ts new file mode 100644 index 000000000..8fad3ffd8 --- /dev/null +++ b/src/lib/apis/projects/index.ts @@ -0,0 +1,167 @@ +import { WEBUI_API_BASE_URL } from '$lib/constants'; + +export const createNewProject = async (token: string, id: string, name: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/projects/create`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + id: id, + name: name + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getProjects = async (token: string = '') => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/projects/`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getProjectById = async (token: string, id: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/projects/${id}`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +type ProjectForm = { + name: string; +}; + +export const updateProjectById = async (token: string, id: string, form: ProjectForm) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/projects/${id}/update`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + name: form.name + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const deleteProjectById = async (token: string, id: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/projects/${id}/delete`, { + method: 'DELETE', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index c10b60aa0..dfdc97511 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -1,10 +1,10 @@ -{#if filteredItems.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')} +{#if filteredProjects.length > 0 || prompt.split(' ')?.at(0)?.substring(1).startsWith('http')}
- {#each filteredItems as doc, docIdx} + {#each filteredProjects as project, idx} {/each} diff --git a/src/lib/components/workspace/Models/Knowledge/Selector.svelte b/src/lib/components/workspace/Models/Knowledge/Selector.svelte index 52d73540e..5dfd43ef9 100644 --- a/src/lib/components/workspace/Models/Knowledge/Selector.svelte +++ b/src/lib/components/workspace/Models/Knowledge/Selector.svelte @@ -1,11 +1,10 @@ diff --git a/src/lib/components/workspace/Projects.svelte b/src/lib/components/workspace/Projects.svelte new file mode 100644 index 000000000..1a37c8282 --- /dev/null +++ b/src/lib/components/workspace/Projects.svelte @@ -0,0 +1,168 @@ + + + + + {$i18n.t('Projects')} | {$WEBUI_NAME} + + + + { + deleteHandler(selectedProject); + }} +/> + +
+
+
+ {$i18n.t('Projects')} +
+ {$projects.length} +
+
+
+ +
+
+
+ + + +
+ +
+ +
+ +
+
+ +
+ +
+ {#each filteredProjects as project} + + {/each} +
+ +
+ ⓘ {$i18n.t("Use '#' in the prompt input to load and select your projects.")} +
diff --git a/src/lib/components/workspace/Projects/CreateProject.svelte b/src/lib/components/workspace/Projects/CreateProject.svelte new file mode 100644 index 000000000..e69de29bb diff --git a/src/lib/components/workspace/Projects/EditProject.svelte b/src/lib/components/workspace/Projects/EditProject.svelte new file mode 100644 index 000000000..e69de29bb diff --git a/src/lib/components/workspace/Projects/ProjectMenu.svelte b/src/lib/components/workspace/Projects/ProjectMenu.svelte new file mode 100644 index 000000000..b21308160 --- /dev/null +++ b/src/lib/components/workspace/Projects/ProjectMenu.svelte @@ -0,0 +1,65 @@ + + + { + if (e.detail === false) { + onClose(); + } + }} + align="end" +> + + + + + +
+ + { + dispatch('delete'); + }} + > + +
{$i18n.t('Delete')}
+ + +
+ diff --git a/src/lib/stores/index.ts b/src/lib/stores/index.ts index b96bf4a98..844eea514 100644 --- a/src/lib/stores/index.ts +++ b/src/lib/stores/index.ts @@ -29,7 +29,7 @@ export const tags = writable([]); export const models: Writable = writable([]); export const prompts: Writable = writable([]); -export const documents: Writable = writable([]); +export const projects: Writable = writable([]); export const tools = writable([]); export const functions = writable([]); diff --git a/src/routes/(app)/+layout.svelte b/src/routes/(app)/+layout.svelte index 83a53dffd..0556a87a1 100644 --- a/src/routes/(app)/+layout.svelte +++ b/src/routes/(app)/+layout.svelte @@ -3,50 +3,46 @@ import { onMount, tick, getContext } from 'svelte'; import { openDB, deleteDB } from 'idb'; import fileSaver from 'file-saver'; + const { saveAs } = fileSaver; import mermaid from 'mermaid'; - const { saveAs } = fileSaver; - import { goto } from '$app/navigation'; + import { page } from '$app/stores'; + import { fade } from 'svelte/transition'; + import { getProjects } from '$lib/apis/projects'; + import { getFunctions } from '$lib/apis/functions'; import { getModels as _getModels, getVersionUpdates } from '$lib/apis'; import { getAllChatTags } from '$lib/apis/chats'; - import { getPrompts } from '$lib/apis/prompts'; - import { getDocs } from '$lib/apis/documents'; import { getTools } from '$lib/apis/tools'; - import { getBanners } from '$lib/apis/configs'; import { getUserSettings } from '$lib/apis/users'; - import { - user, - showSettings, - settings, - models, - prompts, - documents, - tags, - banners, - showChangelog, - config, - showCallOverlay, - tools, - functions, - temporaryChatEnabled - } from '$lib/stores'; - - import SettingsModal from '$lib/components/chat/SettingsModal.svelte'; - import Sidebar from '$lib/components/layout/Sidebar.svelte'; - import ChangelogModal from '$lib/components/ChangelogModal.svelte'; - import AccountPending from '$lib/components/layout/Overlay/AccountPending.svelte'; - import { getFunctions } from '$lib/apis/functions'; - import { page } from '$app/stores'; import { WEBUI_VERSION } from '$lib/constants'; import { compareVersion } from '$lib/utils'; + import { + config, + user, + settings, + models, + prompts, + projects, + tools, + functions, + tags, + banners, + showSettings, + showChangelog, + temporaryChatEnabled + } from '$lib/stores'; + + import Sidebar from '$lib/components/layout/Sidebar.svelte'; + import SettingsModal from '$lib/components/chat/SettingsModal.svelte'; + import ChangelogModal from '$lib/components/ChangelogModal.svelte'; + import AccountPending from '$lib/components/layout/Overlay/AccountPending.svelte'; import UpdateInfoToast from '$lib/components/layout/UpdateInfoToast.svelte'; - import { fade } from 'svelte/transition'; const i18n = getContext('i18n'); @@ -109,7 +105,7 @@ prompts.set(await getPrompts(localStorage.token)); })(), (async () => { - documents.set(await getDocs(localStorage.token)); + projects.set(await getProjects(localStorage.token)); })(), (async () => { tools.set(await getTools(localStorage.token)); diff --git a/src/routes/(app)/workspace/+layout.svelte b/src/routes/(app)/workspace/+layout.svelte index 05ab80715..6f69cffec 100644 --- a/src/routes/(app)/workspace/+layout.svelte +++ b/src/routes/(app)/workspace/+layout.svelte @@ -69,14 +69,12 @@ > - {$i18n.t('Documents')} + {$i18n.t('Projects')} - import Documents from '$lib/components/workspace/Documents.svelte'; - - - diff --git a/src/routes/(app)/workspace/projects/+page.svelte b/src/routes/(app)/workspace/projects/+page.svelte new file mode 100644 index 000000000..9f3f25017 --- /dev/null +++ b/src/routes/(app)/workspace/projects/+page.svelte @@ -0,0 +1,5 @@ + + + diff --git a/src/routes/(app)/workspace/projects/create/+page.svelte b/src/routes/(app)/workspace/projects/create/+page.svelte new file mode 100644 index 000000000..d3744383a --- /dev/null +++ b/src/routes/(app)/workspace/projects/create/+page.svelte @@ -0,0 +1,5 @@ + + + diff --git a/src/routes/(app)/workspace/projects/edit/+page.svelte b/src/routes/(app)/workspace/projects/edit/+page.svelte new file mode 100644 index 000000000..121a5dfcf --- /dev/null +++ b/src/routes/(app)/workspace/projects/edit/+page.svelte @@ -0,0 +1,5 @@ + + + From c2732a099081f9632ca8c4c511b77ffba0a96c5e Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 1 Oct 2024 17:46:56 -0700 Subject: [PATCH 07/45] refac --- .../versions/6a39f3d8e55c_add_project_table.py | 5 +++++ .../chat/MessageInput/Commands/Projects.svelte | 9 --------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/backend/open_webui/migrations/versions/6a39f3d8e55c_add_project_table.py b/backend/open_webui/migrations/versions/6a39f3d8e55c_add_project_table.py index c6c42f64b..201dd1175 100644 --- a/backend/open_webui/migrations/versions/6a39f3d8e55c_add_project_table.py +++ b/backend/open_webui/migrations/versions/6a39f3d8e55c_add_project_table.py @@ -9,6 +9,8 @@ Create Date: 2024-10-01 14:02:35.241684 from alembic import op import sqlalchemy as sa from sqlalchemy.sql import table, column, select +import json + revision = "6a39f3d8e55c" down_revision = "c0fbf31ca0db" @@ -39,6 +41,7 @@ def upgrade(): column("user_id", sa.String()), column("name", sa.String()), column("title", sa.Text()), + column("content", sa.Text()), column("timestamp", sa.BigInteger()), ) @@ -49,6 +52,7 @@ def upgrade(): document_table.c.user_id, document_table.c.name, document_table.c.title, + document_table.c.content, document_table.c.timestamp, ) ) @@ -62,6 +66,7 @@ def upgrade(): description=doc.name, meta={ "legacy": True, + "tags": json.loads(doc.content or "{}").get("tags", []), }, name=doc.title, created_at=doc.timestamp, diff --git a/src/lib/components/chat/MessageInput/Commands/Projects.svelte b/src/lib/components/chat/MessageInput/Commands/Projects.svelte index 9be916b6f..2b80ca0ee 100644 --- a/src/lib/components/chat/MessageInput/Commands/Projects.svelte +++ b/src/lib/components/chat/MessageInput/Commands/Projects.svelte @@ -29,15 +29,6 @@ selectedIdx = 0; } - type ObjectWithName = { - name: string; - }; - - const findByName = (obj: ObjectWithName, command: string) => { - const name = obj.name.toLowerCase(); - return name.includes(command.toLowerCase().split(' ')?.at(0)?.substring(1) ?? ''); - }; - export const selectUp = () => { selectedIdx = Math.max(0, selectedIdx - 1); }; From 1b7d363d32aae5e6583bf2c3b96108d8fcceb18f Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 1 Oct 2024 21:32:59 -0700 Subject: [PATCH 08/45] refac --- .../open_webui/apps/webui/models/projects.py | 3 +- .../open_webui/apps/webui/routers/projects.py | 38 +++-- src/lib/apis/projects/index.ts | 6 +- .../admin/Settings/WebSearch.svelte | 2 +- .../chat/MessageInput/Commands.svelte | 2 +- src/lib/components/workspace/Projects.svelte | 7 +- .../workspace/Projects/CreateProject.svelte | 138 ++++++++++++++++++ .../{EditProject.svelte => Project.svelte} | 0 src/routes/(app)/workspace/+layout.svelte | 2 +- .../workspace/projects/[id]/+page.svelte | 7 + .../workspace/projects/edit/+page.svelte | 5 - 11 files changed, 182 insertions(+), 28 deletions(-) rename src/lib/components/workspace/Projects/{EditProject.svelte => Project.svelte} (100%) create mode 100644 src/routes/(app)/workspace/projects/[id]/+page.svelte delete mode 100644 src/routes/(app)/workspace/projects/edit/+page.svelte diff --git a/backend/open_webui/apps/webui/models/projects.py b/backend/open_webui/apps/webui/models/projects.py index 5b9f07090..4debbbe28 100644 --- a/backend/open_webui/apps/webui/models/projects.py +++ b/backend/open_webui/apps/webui/models/projects.py @@ -2,6 +2,7 @@ import json import logging import time from typing import Optional +import uuid from open_webui.apps.webui.internal.db import Base, get_db from open_webui.env import SRC_LOG_LEVELS @@ -65,7 +66,6 @@ class ProjectResponse(BaseModel): class ProjectForm(BaseModel): - id: str name: str description: str data: Optional[dict] = None @@ -79,6 +79,7 @@ class ProjectTable: project = ProjectModel( **{ **form_data.model_dump(), + "id": str(uuid.uuid4()), "user_id": user_id, "created_at": int(time.time()), "updated_at": int(time.time()), diff --git a/backend/open_webui/apps/webui/routers/projects.py b/backend/open_webui/apps/webui/routers/projects.py index ed47b41b2..493bde99e 100644 --- a/backend/open_webui/apps/webui/routers/projects.py +++ b/backend/open_webui/apps/webui/routers/projects.py @@ -46,21 +46,32 @@ async def get_projects(id: Optional[str] = None, user=Depends(get_verified_user) @router.post("/create", response_model=Optional[ProjectResponse]) async def create_new_project(form_data: ProjectForm, user=Depends(get_admin_user)): - project = Projects.get_project_by_id(form_data.id) - if project is None: - project = Projects.insert_new_project(user.id, form_data) + project = Projects.insert_new_project(user.id, form_data) - if project: - return project - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=ERROR_MESSAGES.FILE_EXISTS, - ) + if project: + return project else: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=ERROR_MESSAGES.ID_TAKEN, + detail=ERROR_MESSAGES.FILE_EXISTS, + ) + + +############################ +# GetProjectById +############################ + + +@router.get("/{id}", response_model=Optional[ProjectResponse]) +async def get_projects(id: str, user=Depends(get_verified_user)): + project = Projects.get_project_by_id(id=id) + + if project: + return project + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=ERROR_MESSAGES.NOT_FOUND, ) @@ -69,8 +80,9 @@ async def create_new_project(form_data: ProjectForm, user=Depends(get_admin_user ############################ -@router.post("/update", response_model=Optional[ProjectResponse]) +@router.post("/{id}/update", response_model=Optional[ProjectResponse]) async def update_project_by_id( + id: str, form_data: ProjectForm, user=Depends(get_admin_user), ): @@ -89,7 +101,7 @@ async def update_project_by_id( ############################ -@router.delete("/delete", response_model=bool) +@router.delete("/{id}/delete", response_model=bool) async def delete_project_by_id(id: str, user=Depends(get_admin_user)): result = Projects.delete_project_by_id(id=id) return result diff --git a/src/lib/apis/projects/index.ts b/src/lib/apis/projects/index.ts index 8fad3ffd8..af448d1bb 100644 --- a/src/lib/apis/projects/index.ts +++ b/src/lib/apis/projects/index.ts @@ -1,6 +1,6 @@ import { WEBUI_API_BASE_URL } from '$lib/constants'; -export const createNewProject = async (token: string, id: string, name: string) => { +export const createNewProject = async (token: string, name: string, description: string) => { let error = null; const res = await fetch(`${WEBUI_API_BASE_URL}/projects/create`, { @@ -11,8 +11,8 @@ export const createNewProject = async (token: string, id: string, name: string) authorization: `Bearer ${token}` }, body: JSON.stringify({ - id: id, - name: name + name: name, + description: description }) }) .then(async (res) => { diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 0a0c2eb16..ddda39b10 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -2,7 +2,7 @@ import { getRAGConfig, updateRAGConfig } from '$lib/apis/retrieval'; import Switch from '$lib/components/common/Switch.svelte'; - import { documents, models } from '$lib/stores'; + import { models } from '$lib/stores'; import { onMount, getContext } from 'svelte'; import { toast } from 'svelte-sonner'; import SensitiveInput from '$lib/components/common/SensitiveInput.svelte'; diff --git a/src/lib/components/chat/MessageInput/Commands.svelte b/src/lib/components/chat/MessageInput/Commands.svelte index bb153f647..f9c0cc25f 100644 --- a/src/lib/components/chat/MessageInput/Commands.svelte +++ b/src/lib/components/chat/MessageInput/Commands.svelte @@ -114,7 +114,7 @@ files = [ ...files, { - type: e?.detail?.type ?? 'file', + type: e?.detail?.meta?.legacy ? 'file' : 'project', ...e.detail, status: 'processed' } diff --git a/src/lib/components/workspace/Projects.svelte b/src/lib/components/workspace/Projects.svelte index 1a37c8282..0eda768f4 100644 --- a/src/lib/components/workspace/Projects.svelte +++ b/src/lib/components/workspace/Projects.svelte @@ -111,12 +111,13 @@
-
+
{#each filteredProjects as project} + {JSON.stringify(project)} + +
{ + submitHandler(); + }} + > +
+
Create a project
+ +
+
+
What are you working on?
+ +
+ +
+
+ +
+
What are you trying to achieve?
+ +
+