From 54b693d5b1811239ca34144f026f8d4914d00317 Mon Sep 17 00:00:00 2001 From: Garfield Dai Date: Wed, 28 Aug 2024 18:55:47 +0800 Subject: [PATCH 01/27] feat: update saas billing hint. (#7760) --- api/controllers/console/datasets/file.py | 2 +- api/controllers/console/wraps.py | 26 +++++++++++------------- api/controllers/service_api/wraps.py | 22 ++++++++------------ 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/api/controllers/console/datasets/file.py b/api/controllers/console/datasets/file.py index d6a464545e..846aa70e86 100644 --- a/api/controllers/console/datasets/file.py +++ b/api/controllers/console/datasets/file.py @@ -39,7 +39,7 @@ class FileApi(Resource): @login_required @account_initialization_required @marshal_with(file_fields) - @cloud_edition_billing_resource_check(resource="documents") + @cloud_edition_billing_resource_check("documents") def post(self): # get file from request file = request.files["file"] diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 5a964c84fa..7667b30e34 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -46,9 +46,7 @@ def only_edition_self_hosted(view): return decorated -def cloud_edition_billing_resource_check( - resource: str, error_msg: str = "You have reached the limit of your subscription." -): +def cloud_edition_billing_resource_check(resource: str): def interceptor(view): @wraps(view) def decorated(*args, **kwargs): @@ -60,22 +58,22 @@ def cloud_edition_billing_resource_check( documents_upload_quota = features.documents_upload_quota annotation_quota_limit = features.annotation_quota_limit if resource == "members" and 0 < members.limit <= members.size: - abort(403, error_msg) + abort(403, "The number of members has reached the limit of your subscription.") elif resource == "apps" and 0 < apps.limit <= apps.size: - abort(403, error_msg) + abort(403, "The number of apps has reached the limit of your subscription.") elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: - abort(403, error_msg) + abort(403, "The capacity of the vector space has reached the limit of your subscription.") elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: # The api of file upload is used in the multiple places, so we need to check the source of the request from datasets source = request.args.get("source") if source == "datasets": - abort(403, error_msg) + abort(403, "The number of documents has reached the limit of your subscription.") else: return view(*args, **kwargs) elif resource == "workspace_custom" and not features.can_replace_logo: - abort(403, error_msg) + abort(403, "The workspace custom feature has reached the limit of your subscription.") elif resource == "annotation" and 0 < annotation_quota_limit.limit < annotation_quota_limit.size: - abort(403, error_msg) + abort(403, "The annotation quota has reached the limit of your subscription.") else: return view(*args, **kwargs) @@ -86,10 +84,7 @@ def cloud_edition_billing_resource_check( return interceptor -def cloud_edition_billing_knowledge_limit_check( - resource: str, - error_msg: str = "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan.", -): +def cloud_edition_billing_knowledge_limit_check(resource: str): def interceptor(view): @wraps(view) def decorated(*args, **kwargs): @@ -97,7 +92,10 @@ def cloud_edition_billing_knowledge_limit_check( if features.billing.enabled: if resource == "add_segment": if features.billing.subscription.plan == "sandbox": - abort(403, error_msg) + abort( + 403, + "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan.", + ) else: return view(*args, **kwargs) diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index a596c6f287..b935b23ed6 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -83,9 +83,7 @@ def validate_app_token(view: Optional[Callable] = None, *, fetch_user_arg: Optio return decorator(view) -def cloud_edition_billing_resource_check( - resource: str, api_token_type: str, error_msg: str = "You have reached the limit of your subscription." -): +def cloud_edition_billing_resource_check(resource: str, api_token_type: str): def interceptor(view): def decorated(*args, **kwargs): api_token = validate_and_get_api_token(api_token_type) @@ -98,13 +96,13 @@ def cloud_edition_billing_resource_check( documents_upload_quota = features.documents_upload_quota if resource == "members" and 0 < members.limit <= members.size: - raise Forbidden(error_msg) + raise Forbidden("The number of members has reached the limit of your subscription.") elif resource == "apps" and 0 < apps.limit <= apps.size: - raise Forbidden(error_msg) + raise Forbidden("The number of apps has reached the limit of your subscription.") elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: - raise Forbidden(error_msg) + raise Forbidden("The capacity of the vector space has reached the limit of your subscription.") elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: - raise Forbidden(error_msg) + raise Forbidden("The number of documents has reached the limit of your subscription.") else: return view(*args, **kwargs) @@ -115,11 +113,7 @@ def cloud_edition_billing_resource_check( return interceptor -def cloud_edition_billing_knowledge_limit_check( - resource: str, - api_token_type: str, - error_msg: str = "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan.", -): +def cloud_edition_billing_knowledge_limit_check(resource: str, api_token_type: str): def interceptor(view): @wraps(view) def decorated(*args, **kwargs): @@ -128,7 +122,9 @@ def cloud_edition_billing_knowledge_limit_check( if features.billing.enabled: if resource == "add_segment": if features.billing.subscription.plan == "sandbox": - raise Forbidden(error_msg) + raise Forbidden( + "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan." + ) else: return view(*args, **kwargs) From 0e0a70349668a9c40b6835a1a5a016a4fdec423a Mon Sep 17 00:00:00 2001 From: takatost Date: Wed, 28 Aug 2024 23:26:11 +0800 Subject: [PATCH 02/27] chore: ignore openai error record in sentry (#7770) --- api/extensions/ext_sentry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/extensions/ext_sentry.py b/api/extensions/ext_sentry.py index 227c6635f0..3b7b0a37f4 100644 --- a/api/extensions/ext_sentry.py +++ b/api/extensions/ext_sentry.py @@ -1,3 +1,4 @@ +import openai import sentry_sdk from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.flask import FlaskIntegration @@ -9,7 +10,7 @@ def init_app(app): sentry_sdk.init( dsn=app.config.get("SENTRY_DSN"), integrations=[FlaskIntegration(), CeleryIntegration()], - ignore_errors=[HTTPException, ValueError], + ignore_errors=[HTTPException, ValueError, openai.APIStatusError], traces_sample_rate=app.config.get("SENTRY_TRACES_SAMPLE_RATE", 1.0), profiles_sample_rate=app.config.get("SENTRY_PROFILES_SAMPLE_RATE", 1.0), environment=app.config.get("DEPLOY_ENV"), From ec1408346eb121f72bfcba9e0f4018efcd31b849 Mon Sep 17 00:00:00 2001 From: kanoshiou <73424326+kanoshiou@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:18:49 +0800 Subject: [PATCH 03/27] docs: navigate to open issues in contributing documents (#7781) --- CONTRIBUTING.md | 2 +- CONTRIBUTING_CN.md | 2 +- CONTRIBUTING_JA.md | 2 +- CONTRIBUTING_VI.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f810584f24..8f57cd545e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ In terms of licensing, please take a minute to read our short [License and Contr ## Before you jump in -[Find](https://github.com/langgenius/dify/issues?q=is:issue+is:closed) an existing issue, or [open](https://github.com/langgenius/dify/issues/new/choose) a new one. We categorize issues into 2 types: +[Find](https://github.com/langgenius/dify/issues?q=is:issue+is:open) an existing issue, or [open](https://github.com/langgenius/dify/issues/new/choose) a new one. We categorize issues into 2 types: ### Feature requests: diff --git a/CONTRIBUTING_CN.md b/CONTRIBUTING_CN.md index 303c2513f5..7cd2bb60eb 100644 --- a/CONTRIBUTING_CN.md +++ b/CONTRIBUTING_CN.md @@ -8,7 +8,7 @@ ## 在开始之前 -[查找](https://github.com/langgenius/dify/issues?q=is:issue+is:closed)现有问题，或 [创建](https://github.com/langgenius/dify/issues/new/choose) 一个新问题。我们将问题分为两类： +[查找](https://github.com/langgenius/dify/issues?q=is:issue+is:open)现有问题，或 [创建](https://github.com/langgenius/dify/issues/new/choose) 一个新问题。我们将问题分为两类： ### 功能请求： diff --git a/CONTRIBUTING_JA.md b/CONTRIBUTING_JA.md index 6d5bfb205c..a68bdeddbc 100644 --- a/CONTRIBUTING_JA.md +++ b/CONTRIBUTING_JA.md @@ -10,7 +10,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは ## 飛び込む前に -[既存の Issue](https://github.com/langgenius/dify/issues?q=is:issue+is:closed) を探すか、[新しい Issue](https://github.com/langgenius/dify/issues/new/choose) を作成してください。私たちは Issue を 2 つのタイプに分類しています。 +[既存の Issue](https://github.com/langgenius/dify/issues?q=is:issue+is:open) を探すか、[新しい Issue](https://github.com/langgenius/dify/issues/new/choose) を作成してください。私たちは Issue を 2 つのタイプに分類しています。 ### 機能リクエスト diff --git a/CONTRIBUTING_VI.md b/CONTRIBUTING_VI.md index 2521256d24..80e68a046e 100644 --- a/CONTRIBUTING_VI.md +++ b/CONTRIBUTING_VI.md @@ -8,7 +8,7 @@ Về vấn đề cấp phép, xin vui lòng dành chút thời gian đọc qua [ ## Trước khi bắt đầu -[Tìm kiếm](https://github.com/langgenius/dify/issues?q=is:issue+is:closed) một vấn đề hiện có, hoặc [tạo mới](https://github.com/langgenius/dify/issues/new/choose) một vấn đề. Chúng tôi phân loại các vấn đề thành 2 loại: +[Tìm kiếm](https://github.com/langgenius/dify/issues?q=is:issue+is:open) một vấn đề hiện có, hoặc [tạo mới](https://github.com/langgenius/dify/issues/new/choose) một vấn đề. Chúng tôi phân loại các vấn đề thành 2 loại: ### Yêu cầu tính năng: From 62f48015234ade1a6c9aa7e1befdfb4d90427ba2 Mon Sep 17 00:00:00 2001 From: Huang YunKun Date: Thu, 29 Aug 2024 11:22:39 +0800 Subject: [PATCH 04/27] Update ssrf_proxy related doc link in docker-compose file (#7778) --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index c1ed2ce5f8..a319d6b45a 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -294,7 +294,7 @@ services: # ssrf_proxy server # for more information, please refer to - # https://docs.dify.ai/learn-more/faq/self-host-faq#id-18.-why-is-ssrf_proxy-needed + # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed ssrf_proxy: image: ubuntu/squid:latest restart: always From ad30668eb62c1c023d3426f38f01ee3ceb637b11 Mon Sep 17 00:00:00 2001 From: NFish Date: Thu, 29 Aug 2024 11:23:16 +0800 Subject: [PATCH 05/27] Sync Input component from feat/attachments branch (#7782) --- web/app/components/base/input/index.tsx | 104 ++++++++++++------ .../empty-dataset-creation-modal/index.tsx | 2 +- .../documents/detail/completed/index.tsx | 2 +- .../documents/detail/metadata/index.tsx | 2 +- .../components/datasets/documents/index.tsx | 4 +- .../key-value/key-value-edit/item.tsx | 2 +- 6 files changed, 78 insertions(+), 38 deletions(-) diff --git a/web/app/components/base/input/index.tsx b/web/app/components/base/input/index.tsx index 5ab8249446..cac5efd879 100644 --- a/web/app/components/base/input/index.tsx +++ b/web/app/components/base/input/index.tsx @@ -1,43 +1,83 @@ -'use client' -import type { SVGProps } from 'react' -import React, { useState } from 'react' +import type { CSSProperties } from 'react' +import React from 'react' import { useTranslation } from 'react-i18next' -import cn from 'classnames' +import { RiCloseCircleFill, RiErrorWarningLine, RiSearchLine } from '@remixicon/react' +import { type VariantProps, cva } from 'class-variance-authority' +import cn from '@/utils/classnames' -type InputProps = { - placeholder?: string - value?: string - defaultValue?: string - onChange?: (v: string) => void - className?: string - wrapperClassName?: string - type?: string - showPrefix?: React.ReactNode - prefixIcon?: React.ReactNode -} - -const GlassIcon = ({ className }: SVGProps) => ( - - - +export const inputVariants = cva( + '', + { + variants: { + size: { + regular: 'px-3 radius-md system-sm-regular', + large: 'px-4 radius-lg system-md-regular', + }, + }, + defaultVariants: { + size: 'regular', + }, + }, ) -const Input = ({ value, defaultValue, onChange, className = '', wrapperClassName = '', placeholder, type, showPrefix, prefixIcon }: InputProps) => { - const [localValue, setLocalValue] = useState(value ?? defaultValue) +export type InputProps = { + showLeftIcon?: boolean + showClearIcon?: boolean + onClear?: () => void + disabled?: boolean + destructive?: boolean + wrapperClassName?: string + styleCss?: CSSProperties +} & React.InputHTMLAttributes & VariantProps + +const Input = ({ + size, + disabled, + destructive, + showLeftIcon, + showClearIcon, + onClear, + wrapperClassName, + className, + styleCss, + value, + placeholder, + onChange, + ...props +}: InputProps) => { const { t } = useTranslation() return ( -
- {showPrefix && {prefixIcon ?? }} +
+ {showLeftIcon && } { - setLocalValue(e.target.value) - onChange && onChange(e.target.value) - }} + style={styleCss} + className={cn( + 'w-full py-[7px] bg-components-input-bg-normal border border-transparent text-components-input-text-filled hover:bg-components-input-bg-hover hover:border-components-input-border-hover focus:bg-components-input-bg-active focus:border-components-input-border-active focus:shadow-xs placeholder:text-components-input-text-placeholder appearance-none outline-none caret-primary-600', + inputVariants({ size }), + showLeftIcon && 'pl-[26px]', + showLeftIcon && size === 'large' && 'pl-7', + showClearIcon && value && 'pr-[26px]', + showClearIcon && value && size === 'large' && 'pr-7', + destructive && 'pr-[26px]', + destructive && size === 'large' && 'pr-7', + disabled && 'bg-components-input-bg-disabled border-transparent text-components-input-text-filled-disabled cursor-not-allowed hover:bg-components-input-bg-disabled hover:border-transparent', + destructive && 'bg-components-input-bg-destructive border-components-input-border-destructive text-components-input-text-filled hover:bg-components-input-bg-destructive hover:border-components-input-border-destructive focus:bg-components-input-bg-destructive focus:border-components-input-border-destructive', + className, + )} + placeholder={placeholder ?? (showLeftIcon ? t('common.operation.search') ?? '' : 'please input')} + value={value} + onChange={onChange} + disabled={disabled} + {...props} /> + {showClearIcon && value && !disabled && !destructive && ( +
+ +
+ )} + {destructive && ( + + )}
) } diff --git a/web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx b/web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx index e9247c49df..c490d50b88 100644 --- a/web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx +++ b/web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx @@ -58,7 +58,7 @@ const EmptyDatasetCreationModal = ({
{t('datasetCreation.stepOne.modal.tip')}
{t('datasetCreation.stepOne.modal.input')}
- + setInputValue(e.target.value)} />
diff --git a/web/app/components/datasets/documents/detail/completed/index.tsx b/web/app/components/datasets/documents/detail/completed/index.tsx index f2addac2e2..195b75b0a2 100644 --- a/web/app/components/datasets/documents/detail/completed/index.tsx +++ b/web/app/components/datasets/documents/detail/completed/index.tsx @@ -391,7 +391,7 @@ const Completed: FC = ({ defaultValue={'all'} className={s.select} wrapperClassName='h-fit w-[120px] mr-2' /> - + setSearchValue(e.target.value), 500)} />
= ({ /> : onUpdate?.(e.target.value)} value={value} defaultValue={defaultValue} placeholder={`${t('datasetDocuments.metadata.placeholder.add')}${label}`} diff --git a/web/app/components/datasets/documents/index.tsx b/web/app/components/datasets/documents/index.tsx index ce81e44792..32c045eccd 100644 --- a/web/app/components/datasets/documents/index.tsx +++ b/web/app/components/datasets/documents/index.tsx @@ -201,10 +201,10 @@ const Documents: FC = ({ datasetId }) => {
setSearchValue(e.target.value), 500)} value={searchValue} />
diff --git a/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/item.tsx b/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/item.tsx index 93c2696b98..0f28db96c3 100644 --- a/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/item.tsx +++ b/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/item.tsx @@ -73,7 +73,7 @@ const KeyValueItem: FC = ({ handleChange('key')(e.target.value)} /> )}
From c441bea4d14312fc1c535e63e67cbbfb3ea0c368 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:46:33 +0800 Subject: [PATCH 06/27] fix: datasets permission is missing (#7787) --- api/controllers/console/datasets/datasets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index d369730594..44c1390c14 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -122,6 +122,7 @@ class DatasetListApi(Resource): name=args["name"], indexing_technique=args["indexing_technique"], account=current_user, + permission=DatasetPermissionEnum.ONLY_ME, ) except services.errors.dataset.DatasetNameDuplicateError: raise DatasetNameDuplicateError() From 8e311cc45cfeabf2fc88527619fef1c8337be94a Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:46:42 +0800 Subject: [PATCH 07/27] fixed permission is None (#7788) --- api/services/dataset_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index ce5fa632dc..d10df3f600 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -137,7 +137,7 @@ class DatasetService: @staticmethod def create_empty_dataset( - tenant_id: str, name: str, indexing_technique: Optional[str], account: Account, permission: Optional[str] + tenant_id: str, name: str, indexing_technique: Optional[str], account: Account, permission: Optional[str] = None ): # check if dataset name already exists if Dataset.query.filter_by(name=name, tenant_id=tenant_id).first(): From 2c51e3a3275092603ba5f9548e4aea7f5e99ebf7 Mon Sep 17 00:00:00 2001 From: NFish Date: Thu, 29 Aug 2024 15:57:43 +0800 Subject: [PATCH 08/27] fix: webapp sso setting may not the latest value when refresh (#7795) --- .../app/(appDetailLayout)/[appId]/layout.tsx | 2 +- .../app/(appDetailLayout)/[appId]/overview/cardView.tsx | 2 +- web/app/components/app/overview/appCard.tsx | 9 ++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx index 84ec157323..8723420d84 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx @@ -128,7 +128,7 @@ const AppDetailLayout: FC = (props) => { if (e.status === 404) router.replace('/apps') }) - }, [appId, isCurrentWorkspaceEditor]) + }, [appId, isCurrentWorkspaceEditor, systemFeatures]) useUnmount(() => { setAppDetail() diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx index 3584e13733..8f3ee510b8 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx @@ -95,7 +95,7 @@ const CardView: FC = ({ appId }) => { if (systemFeatures.enable_web_sso_switch_component) { const [sso_err] = await asyncRunSafe( - updateAppSSO({ id: appId, enabled: params.enable_sso }) as Promise, + updateAppSSO({ id: appId, enabled: Boolean(params.enable_sso) }) as Promise, ) if (sso_err) { handleCallbackResult(sso_err) diff --git a/web/app/components/app/overview/appCard.tsx b/web/app/components/app/overview/appCard.tsx index 0d0b95c0a0..f9f5c1fbff 100644 --- a/web/app/components/app/overview/appCard.tsx +++ b/web/app/components/app/overview/appCard.tsx @@ -134,8 +134,8 @@ function AppCard({ return (
@@ -176,7 +176,6 @@ function AppCard({ {isApp && } {/* button copy link/ button regenerate */} @@ -202,8 +201,8 @@ function AppCard({ onClick={() => setShowConfirmDelete(true)} >
From 962cdbbebd9e0614ea05593aed785faa063d7cf4 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:04:01 +0800 Subject: [PATCH 09/27] chore: add app generator overload (#7792) --- .../app/apps/advanced_chat/app_generator.py | 22 ++++++++++++++++- api/core/app/apps/agent_chat/app_generator.py | 20 +++++++++++++++- api/core/app/apps/chat/app_generator.py | 22 +++++++++++++++-- api/core/app/apps/completion/app_generator.py | 24 ++++++++++++++++--- api/core/app/apps/workflow/app_generator.py | 24 +++++++++++++++++-- 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 5a1e5973cd..e7c9ebe097 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -4,7 +4,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Union +from typing import Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -39,6 +39,26 @@ logger = logging.getLogger(__name__) class AdvancedChatAppGenerator(MessageBasedAppGenerator): + @overload + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[str, None, None]: ... + + @overload + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate( self, app_model: App, workflow: Workflow, diff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.py index 53780bdfb0..daf37f4a50 100644 --- a/api/core/app/apps/agent_chat/app_generator.py +++ b/api/core/app/apps/agent_chat/app_generator.py @@ -3,7 +3,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Any, Union +from typing import Any, Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -28,6 +28,24 @@ logger = logging.getLogger(__name__) class AgentChatAppGenerator(MessageBasedAppGenerator): + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[dict, None, None]: ... + + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate(self, app_model: App, user: Union[Account, EndUser], args: Any, diff --git a/api/core/app/apps/chat/app_generator.py b/api/core/app/apps/chat/app_generator.py index 5b896e2845..ab15928b74 100644 --- a/api/core/app/apps/chat/app_generator.py +++ b/api/core/app/apps/chat/app_generator.py @@ -3,7 +3,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Any, Union +from typing import Any, Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -28,13 +28,31 @@ logger = logging.getLogger(__name__) class ChatAppGenerator(MessageBasedAppGenerator): + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: Any, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[str, None, None]: ... + + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: Any, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate( self, app_model: App, user: Union[Account, EndUser], args: Any, invoke_from: InvokeFrom, stream: bool = True, - ) -> Union[dict, Generator[dict, None, None]]: + ) -> Union[dict, Generator[str, None, None]]: """ Generate App response. diff --git a/api/core/app/apps/completion/app_generator.py b/api/core/app/apps/completion/app_generator.py index c4e1caf65a..c0b13b40fd 100644 --- a/api/core/app/apps/completion/app_generator.py +++ b/api/core/app/apps/completion/app_generator.py @@ -3,7 +3,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Any, Union +from typing import Any, Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -30,12 +30,30 @@ logger = logging.getLogger(__name__) class CompletionAppGenerator(MessageBasedAppGenerator): + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[str, None, None]: ... + + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate(self, app_model: App, user: Union[Account, EndUser], args: Any, invoke_from: InvokeFrom, stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + -> Union[dict, Generator[str, None, None]]: """ Generate App response. @@ -203,7 +221,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator): user: Union[Account, EndUser], invoke_from: InvokeFrom, stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + -> Union[dict, Generator[str, None, None]]: """ Generate App response. diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index df40aec154..26bb6c0f4f 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -4,7 +4,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Union +from typing import Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -32,6 +32,26 @@ logger = logging.getLogger(__name__) class WorkflowAppGenerator(BaseAppGenerator): + @overload + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[str, None, None]: ... + + @overload + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate( self, app_model: App, workflow: Workflow, @@ -107,7 +127,7 @@ class WorkflowAppGenerator(BaseAppGenerator): application_generate_entity: WorkflowAppGenerateEntity, invoke_from: InvokeFrom, stream: bool = True, - ) -> Union[dict, Generator[dict, None, None]]: + ) -> Union[dict, Generator[str, None, None]]: """ Generate App response. From f0273f00e18d023fd6270307f9eb5c1030567e1b Mon Sep 17 00:00:00 2001 From: hisir Date: Thu, 29 Aug 2024 18:58:19 +0800 Subject: [PATCH 10/27] Fixed when testing the openai compatible interface model, an error is reported when no object is returned (#7808) --- .../model_providers/openai_api_compatible/llm/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py index 2b729d4293..6279125f46 100644 --- a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py +++ b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py @@ -150,9 +150,9 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): except json.JSONDecodeError as e: raise CredentialsValidateFailedError('Credentials validation failed: JSON decode error') - if (completion_type is LLMMode.CHAT and json_result['object'] == ''): + if (completion_type is LLMMode.CHAT and json_result.get('object','') == ''): json_result['object'] = 'chat.completion' - elif (completion_type is LLMMode.COMPLETION and json_result['object'] == ''): + elif (completion_type is LLMMode.COMPLETION and json_result.get('object','') == ''): json_result['object'] = 'text_completion' if (completion_type is LLMMode.CHAT From bd6d4d0553696f711e2f4ab101b952a34c3ca33c Mon Sep 17 00:00:00 2001 From: YidaHu Date: Thu, 29 Aug 2024 19:03:08 +0800 Subject: [PATCH 11/27] fix: filter out installed apps without an app (#7799) --- api/controllers/console/explore/installed_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/controllers/console/explore/installed_app.py b/api/controllers/console/explore/installed_app.py index b71078760c..3f1e64a247 100644 --- a/api/controllers/console/explore/installed_app.py +++ b/api/controllers/console/explore/installed_app.py @@ -35,6 +35,7 @@ class InstalledAppsListApi(Resource): "uninstallable": current_tenant_id == installed_app.app_owner_tenant_id, } for installed_app in installed_apps + if installed_app.app is not None ] installed_apps.sort( key=lambda app: ( From c9e0f0bf20a587e640284a02070b95a1611f37a9 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Fri, 30 Aug 2024 00:03:40 +0800 Subject: [PATCH 12/27] fix: correct typo in environment variable description (#7817) --- web/i18n/zh-Hans/workflow.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/zh-Hans/workflow.ts b/web/i18n/zh-Hans/workflow.ts index f57eb40bb0..56d1de6ceb 100644 --- a/web/i18n/zh-Hans/workflow.ts +++ b/web/i18n/zh-Hans/workflow.ts @@ -80,7 +80,7 @@ const translation = { }, env: { envPanelTitle: '环境变量', - envDescription: '环境变量是一种存储敏感信息的方法，如 API 密钥、数据库密码等。它们被存储在工作流程中，而不是代码中，以便在不同环墋中共享。', + envDescription: '环境变量是一种存储敏感信息的方法，如 API 密钥、数据库密码等。它们被存储在工作流程中，而不是代码中，以便在不同环境中共享。', envPanelButton: '添加环境变量', modal: { title: '添加环境变量', From dc015c380a94543da572222685bcfbf1af766eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= Date: Fri, 30 Aug 2024 15:08:31 +0800 Subject: [PATCH 13/27] feat: add zhipu glm_4_plus and glm_4v_plus model (#7824) --- .../zhipuai/llm/glm_4_plus.yaml | 39 +++++++++++++++++++ .../zhipuai/llm/glm_4v_plus.yaml | 37 ++++++++++++++++++ .../model_providers/zhipuai/llm/llm.py | 9 +++-- 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 api/core/model_runtime/model_providers/zhipuai/llm/glm_4_plus.yaml create mode 100644 api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/glm_4_plus.yaml b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4_plus.yaml new file mode 100644 index 0000000000..d9132640b2 --- /dev/null +++ b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4_plus.yaml @@ -0,0 +1,39 @@ +model: glm-4-plus +label: + en_US: glm-4-plus +model_type: llm +features: + - multi-tool-call + - agent-thought + - stream-tool-call +model_properties: + mode: chat +parameter_rules: + - name: temperature + use_template: temperature + default: 0.95 + min: 0.0 + max: 1.0 + help: + zh_Hans: 采样温度，控制输出的随机性，必须为正数取值范围是：(0.0,1.0]，不能等于 0,默认值为 0.95 值越大，会使输出更随机，更具创造性；值越小，输出会更加稳定或确定建议您根据应用场景调整 top_p 或 temperature 参数，但不要同时调整两个参数。 + en_US: Sampling temperature, controls the randomness of the output, must be a positive number. The value range is (0.0,1.0], which cannot be equal to 0. The default value is 0.95. The larger the value, the more random and creative the output will be; the smaller the value, The output will be more stable or certain. It is recommended that you adjust the top_p or temperature parameters according to the application scenario, but do not adjust both parameters at the same time. + - name: top_p + use_template: top_p + default: 0.7 + help: + zh_Hans: 用温度取样的另一种方法，称为核取样取值范围是：(0.0, 1.0) 开区间，不能等于 0 或 1，默认值为 0.7 模型考虑具有 top_p 概率质量tokens的结果例如：0.1 意味着模型解码器只考虑从前 10% 的概率的候选集中取 tokens 建议您根据应用场景调整 top_p 或 temperature 参数，但不要同时调整两个参数。 + en_US: Another method of temperature sampling is called kernel sampling. The value range is (0.0, 1.0) open interval, which cannot be equal to 0 or 1. The default value is 0.7. The model considers the results with top_p probability mass tokens. For example 0.1 means The model decoder only considers tokens from the candidate set with the top 10% probability. It is recommended that you adjust the top_p or temperature parameters according to the application scenario, but do not adjust both parameters at the same time. + - name: incremental + label: + zh_Hans: 增量返回 + en_US: Incremental + type: boolean + help: + zh_Hans: SSE接口调用时，用于控制每次返回内容方式是增量还是全量，不提供此参数时默认为增量返回，true 为增量返回，false 为全量返回。 + en_US: When the SSE interface is called, it is used to control whether the content is returned incrementally or in full. If this parameter is not provided, the default is incremental return. true means incremental return, false means full return. + required: false + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 8192 diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml new file mode 100644 index 0000000000..4a45c65f64 --- /dev/null +++ b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml @@ -0,0 +1,37 @@ +model: glm-4v-plus +label: + en_US: glm-4v-plus +model_type: llm +model_properties: + mode: chat +features: + - vision +parameter_rules: + - name: temperature + use_template: temperature + default: 0.95 + min: 0.0 + max: 1.0 + help: + zh_Hans: 采样温度，控制输出的随机性，必须为正数取值范围是：(0.0,1.0]，不能等于 0,默认值为 0.95 值越大，会使输出更随机，更具创造性；值越小，输出会更加稳定或确定建议您根据应用场景调整 top_p 或 temperature 参数，但不要同时调整两个参数。 + en_US: Sampling temperature, controls the randomness of the output, must be a positive number. The value range is (0.0,1.0], which cannot be equal to 0. The default value is 0.95. The larger the value, the more random and creative the output will be; the smaller the value, The output will be more stable or certain. It is recommended that you adjust the top_p or temperature parameters according to the application scenario, but do not adjust both parameters at the same time. + - name: top_p + use_template: top_p + default: 0.7 + help: + zh_Hans: 用温度取样的另一种方法，称为核取样取值范围是：(0.0, 1.0) 开区间，不能等于 0 或 1，默认值为 0.7 模型考虑具有 top_p 概率质量tokens的结果例如：0.1 意味着模型解码器只考虑从前 10% 的概率的候选集中取 tokens 建议您根据应用场景调整 top_p 或 temperature 参数，但不要同时调整两个参数。 + en_US: Another method of temperature sampling is called kernel sampling. The value range is (0.0, 1.0) open interval, which cannot be equal to 0 or 1. The default value is 0.7. The model considers the results with top_p probability mass tokens. For example 0.1 means The model decoder only considers tokens from the candidate set with the top 10% probability. It is recommended that you adjust the top_p or temperature parameters according to the application scenario, but do not adjust both parameters at the same time. + - name: incremental + label: + zh_Hans: 增量返回 + en_US: Incremental + type: boolean + help: + zh_Hans: SSE接口调用时，用于控制每次返回内容方式是增量还是全量，不提供此参数时默认为增量返回，true 为增量返回，false 为全量返回。 + en_US: When the SSE interface is called, it is used to control whether the content is returned incrementally or in full. If this parameter is not provided, the default is incremental return. true means incremental return, false means full return. + required: false + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 8192 diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py index 13d8f5e5c3..b2cdc7ad7a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -153,7 +153,8 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): :return: full response or stream response chunk generator result """ extra_model_kwargs = {} - if stop: + # request to glm-4v-plus with stop words will always response "finish_reason":"network_error" + if stop and model!= 'glm-4v-plus': extra_model_kwargs['stop'] = stop client = ZhipuAI( @@ -174,7 +175,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): if copy_prompt_message.role in [PromptMessageRole.USER, PromptMessageRole.SYSTEM, PromptMessageRole.TOOL]: if isinstance(copy_prompt_message.content, list): # check if model is 'glm-4v' - if model != 'glm-4v': + if model not in ('glm-4v', 'glm-4v-plus'): # not support list message continue # get image and @@ -207,7 +208,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): else: new_prompt_messages.append(copy_prompt_message) - if model == 'glm-4v': + if model == 'glm-4v' or model == 'glm-4v-plus': params = self._construct_glm_4v_parameter(model, new_prompt_messages, model_parameters) else: params = { @@ -304,7 +305,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): return params - def _construct_glm_4v_messages(self, prompt_message: Union[str | list[PromptMessageContent]]) -> list[dict]: + def _construct_glm_4v_messages(self, prompt_message: Union[str, list[PromptMessageContent]]) -> list[dict]: if isinstance(prompt_message, str): return [{'type': 'text', 'text': prompt_message}] From ceb2b150fff62fa515ec26e1aeffce0095d5c7be Mon Sep 17 00:00:00 2001 From: Zhi Date: Fri, 30 Aug 2024 15:53:50 +0800 Subject: [PATCH 14/27] enhance: include workspace name in create-tenant command (#7834) --- api/commands.py | 7 +++++-- api/services/account_service.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/api/commands.py b/api/commands.py index 41f1a6444c..3bf8bc0ecc 100644 --- a/api/commands.py +++ b/api/commands.py @@ -559,8 +559,9 @@ def add_qdrant_doc_id_index(field: str): @click.command("create-tenant", help="Create account and tenant.") @click.option("--email", prompt=True, help="The email address of the tenant account.") +@click.option("--name", prompt=True, help="The workspace name of the tenant account.") @click.option("--language", prompt=True, help="Account language, default: en-US.") -def create_tenant(email: str, language: Optional[str] = None): +def create_tenant(email: str, language: Optional[str] = None, name: Optional[str] = None): """ Create tenant account """ @@ -580,13 +581,15 @@ def create_tenant(email: str, language: Optional[str] = None): if language not in languages: language = "en-US" + name = name.strip() + # generate random password new_password = secrets.token_urlsafe(16) # register account account = RegisterService.register(email=email, name=account_name, password=new_password, language=language) - TenantService.create_owner_tenant_if_not_exist(account) + TenantService.create_owner_tenant_if_not_exist(account, name) click.echo( click.style( diff --git a/api/services/account_service.py b/api/services/account_service.py index cd501c9792..e1b70fc9ed 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -265,7 +265,7 @@ class TenantService: return tenant @staticmethod - def create_owner_tenant_if_not_exist(account: Account): + def create_owner_tenant_if_not_exist(account: Account, name: Optional[str] = None): """Create owner tenant if not exist""" available_ta = ( TenantAccountJoin.query.filter_by(account_id=account.id).order_by(TenantAccountJoin.id.asc()).first() @@ -274,7 +274,10 @@ class TenantService: if available_ta: return - tenant = TenantService.create_tenant(f"{account.name}'s Workspace") + if name: + tenant = TenantService.create_tenant(name) + else: + tenant = TenantService.create_tenant(f"{account.name}'s Workspace") TenantService.create_tenant_member(tenant, account, role="owner") account.current_tenant = tenant db.session.commit() From 62bfc4dba6e5e99ddf162b659b4cf07054c16559 Mon Sep 17 00:00:00 2001 From: Yi Xiao <54782454+YIXIAO0@users.noreply.github.com> Date: Fri, 30 Aug 2024 18:13:54 +0800 Subject: [PATCH 15/27] fix: tooltip size sets improperly (#7836) --- .../common/retrieval-param-config/index.tsx | 2 +- .../system-model-selector/index.tsx | 13 +++++-------- .../config-credentials.tsx | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/web/app/components/datasets/common/retrieval-param-config/index.tsx b/web/app/components/datasets/common/retrieval-param-config/index.tsx index 54a0963f59..323e47f3b4 100644 --- a/web/app/components/datasets/common/retrieval-param-config/index.tsx +++ b/web/app/components/datasets/common/retrieval-param-config/index.tsx @@ -191,7 +191,7 @@ const RetrievalParamConfig: FC = ({
{option.label}
{option.tips}
} - triggerClassName='ml-0.5 w-3.5 h-4.5' + triggerClassName='ml-0.5 w-3.5 h-3.5' />
)) diff --git a/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx b/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx index 6e181425e2..1574785898 100644 --- a/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx @@ -148,7 +148,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.systemReasoningModel.tip')}
} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
@@ -168,8 +168,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.embeddingModel.tip')}
} - needsDelay={false} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
@@ -189,8 +188,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.rerankModel.tip')}
} - needsDelay={false} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
@@ -210,8 +208,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.speechToTextModel.tip')}
} - needsDelay={false} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
@@ -231,7 +228,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.ttsModel.tip')}
} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
diff --git a/web/app/components/tools/edit-custom-collection-modal/config-credentials.tsx b/web/app/components/tools/edit-custom-collection-modal/config-credentials.tsx index c7685d496d..d580c00102 100644 --- a/web/app/components/tools/edit-custom-collection-modal/config-credentials.tsx +++ b/web/app/components/tools/edit-custom-collection-modal/config-credentials.tsx @@ -114,7 +114,7 @@ const ConfigCredential: FC = ({ {t('tools.createTool.authMethod.keyTooltip')}
} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4' /> Date: Fri, 30 Aug 2024 18:48:05 +0800 Subject: [PATCH 16/27] fix: an issue of keyword search feature in application log list (#7816) --- api/controllers/console/app/conversation.py | 27 +++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/api/controllers/console/app/conversation.py b/api/controllers/console/app/conversation.py index 753a6be20c..c3aac6690e 100644 --- a/api/controllers/console/app/conversation.py +++ b/api/controllers/console/app/conversation.py @@ -173,21 +173,18 @@ class ChatConversationApi(Resource): if args["keyword"]: keyword_filter = "%{}%".format(args["keyword"]) - query = ( - query.join( - Message, - Message.conversation_id == Conversation.id, - ) - .join(subquery, subquery.c.conversation_id == Conversation.id) - .filter( - or_( - Message.query.ilike(keyword_filter), - Message.answer.ilike(keyword_filter), - Conversation.name.ilike(keyword_filter), - Conversation.introduction.ilike(keyword_filter), - subquery.c.from_end_user_session_id.ilike(keyword_filter), - ), - ) + message_subquery = ( + db.session.query(Message.conversation_id) + .filter(or_(Message.query.ilike(keyword_filter), Message.answer.ilike(keyword_filter))) + .subquery() + ) + query = query.join(subquery, subquery.c.conversation_id == Conversation.id).filter( + or_( + Conversation.id.in_(message_subquery), + Conversation.name.ilike(keyword_filter), + Conversation.introduction.ilike(keyword_filter), + subquery.c.from_end_user_session_id.ilike(keyword_filter), + ), ) account = current_user From 0c2a62f8479d4cfcc18751b5859290af7d24370b Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 30 Aug 2024 19:09:10 +0800 Subject: [PATCH 17/27] =?UTF-8?q?fix:=20correct=20http=20timeout=20configs?= =?UTF-8?q?=E2=80=98=20default=20values=20and=20ignorance=20by=20HttpReque?= =?UTF-8?q?stNode=20(#7762)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/configs/feature/__init__.py | 23 ++++++++----------- .../nodes/http_request/http_request_node.py | 15 +++++------- .../unit_tests/configs/test_dify_config.py | 7 ++++++ 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index f2efa52de3..303bce2aa5 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Annotated, Optional from pydantic import AliasChoices, Field, HttpUrl, NegativeInt, NonNegativeInt, PositiveInt, computed_field from pydantic_settings import BaseSettings @@ -217,20 +217,17 @@ class HttpConfig(BaseSettings): def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]: return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(",") - HTTP_REQUEST_MAX_CONNECT_TIMEOUT: NonNegativeInt = Field( - description="", - default=300, - ) + HTTP_REQUEST_MAX_CONNECT_TIMEOUT: Annotated[ + PositiveInt, Field(ge=10, description="connect timeout in seconds for HTTP request") + ] = 10 - HTTP_REQUEST_MAX_READ_TIMEOUT: NonNegativeInt = Field( - description="", - default=600, - ) + HTTP_REQUEST_MAX_READ_TIMEOUT: Annotated[ + PositiveInt, Field(ge=60, description="read timeout in seconds for HTTP request") + ] = 60 - HTTP_REQUEST_MAX_WRITE_TIMEOUT: NonNegativeInt = Field( - description="", - default=600, - ) + HTTP_REQUEST_MAX_WRITE_TIMEOUT: Annotated[ + PositiveInt, Field(ge=10, description="read timeout in seconds for HTTP request") + ] = 20 HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field( description="", diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index f6c8ea3c83..037a7a1848 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -19,9 +19,9 @@ from core.workflow.nodes.http_request.http_executor import HttpExecutor, HttpExe from models.workflow import WorkflowNodeExecutionStatus HTTP_REQUEST_DEFAULT_TIMEOUT = HttpRequestNodeTimeout( - connect=min(10, dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT), - read=min(60, dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT), - write=min(20, dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT), + connect=dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT, + read=dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT, + write=dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT, ) @@ -96,12 +96,9 @@ class HttpRequestNode(BaseNode): if timeout is None: return HTTP_REQUEST_DEFAULT_TIMEOUT - timeout.connect = min(timeout.connect or HTTP_REQUEST_DEFAULT_TIMEOUT.connect, - dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT) - timeout.read = min(timeout.read or HTTP_REQUEST_DEFAULT_TIMEOUT.read, - dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT) - timeout.write = min(timeout.write or HTTP_REQUEST_DEFAULT_TIMEOUT.write, - dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT) + timeout.connect = timeout.connect or HTTP_REQUEST_DEFAULT_TIMEOUT.connect + timeout.read = timeout.read or HTTP_REQUEST_DEFAULT_TIMEOUT.read + timeout.write = timeout.write or HTTP_REQUEST_DEFAULT_TIMEOUT.write return timeout @classmethod diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.py index fb415483dd..3f639ccacc 100644 --- a/api/tests/unit_tests/configs/test_dify_config.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -19,6 +19,7 @@ def example_env_file(tmp_path, monkeypatch) -> str: """ CONSOLE_API_URL=https://example.com CONSOLE_WEB_URL=https://example.com + HTTP_REQUEST_MAX_WRITE_TIMEOUT=30 """ ) ) @@ -48,6 +49,12 @@ def test_dify_config(example_env_file): assert config.API_COMPRESSION_ENABLED is False assert config.SENTRY_TRACES_SAMPLE_RATE == 1.0 + # annotated field with default value + assert config.HTTP_REQUEST_MAX_READ_TIMEOUT == 60 + + # annotated field with configured value + assert config.HTTP_REQUEST_MAX_WRITE_TIMEOUT == 30 + # NOTE: If there is a `.env` file in your Workspace, this test might not succeed as expected. # This is due to `pymilvus` loading all the variables from the `.env` file into `os.environ`. From 1f56a20b626d2e721404ecab179fac81a5dffe90 Mon Sep 17 00:00:00 2001 From: sino Date: Sat, 31 Aug 2024 10:56:32 +0800 Subject: [PATCH 18/27] feat: support auth by api key for ark provider (#7845) --- .../model_providers/volcengine_maas/client.py | 19 +++++++++-- .../volcengine_maas/volcengine_maas.yaml | 34 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/api/core/model_runtime/model_providers/volcengine_maas/client.py b/api/core/model_runtime/model_providers/volcengine_maas/client.py index 61f3521a43..a4d89dabcb 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/client.py +++ b/api/core/model_runtime/model_providers/volcengine_maas/client.py @@ -71,11 +71,24 @@ class ArkClientV3: args = { "base_url": credentials['api_endpoint_host'], "region": credentials['volc_region'], - "ak": credentials['volc_access_key_id'], - "sk": credentials['volc_secret_access_key'], } + if credentials.get("auth_method") == "api_key": + args = { + **args, + "api_key": credentials['volc_api_key'], + } + else: + args = { + **args, + "ak": credentials['volc_access_key_id'], + "sk": credentials['volc_secret_access_key'], + } + if cls.is_compatible_with_legacy(credentials): - args["base_url"] = DEFAULT_V3_ENDPOINT + args = { + **args, + "base_url": DEFAULT_V3_ENDPOINT + } client = ArkClientV3( **args diff --git a/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml b/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml index 735ba2b314..13e00da76f 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml +++ b/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml @@ -30,8 +30,28 @@ model_credential_schema: en_US: Enter your Model Name zh_Hans: 输入模型名称 credential_form_schemas: + - variable: auth_method + required: true + label: + en_US: Authentication Method + zh_Hans: 鉴权方式 + type: select + default: aksk + options: + - label: + en_US: API Key + value: api_key + - label: + en_US: Access Key / Secret Access Key + value: aksk + placeholder: + en_US: Enter your Authentication Method + zh_Hans: 选择鉴权方式 - variable: volc_access_key_id required: true + show_on: + - variable: auth_method + value: aksk label: en_US: Access Key zh_Hans: Access Key @@ -41,6 +61,9 @@ model_credential_schema: zh_Hans: 输入您的 Access Key - variable: volc_secret_access_key required: true + show_on: + - variable: auth_method + value: aksk label: en_US: Secret Access Key zh_Hans: Secret Access Key @@ -48,6 +71,17 @@ model_credential_schema: placeholder: en_US: Enter your Secret Access Key zh_Hans: 输入您的 Secret Access Key + - variable: volc_api_key + required: true + show_on: + - variable: auth_method + value: api_key + label: + en_US: API Key + type: secret-input + placeholder: + en_US: Enter your API Key + zh_Hans: 输入您的 API Key - variable: volc_region required: true label: From ef82a29e2307f08c3ce6ce51720afc43e2d5bfc1 Mon Sep 17 00:00:00 2001 From: zhujinle <32472522+zhujinle@users.noreply.github.com> Date: Sun, 1 Sep 2024 14:52:27 +0800 Subject: [PATCH 19/27] fix: crash when ECharts accesses undefined objects (#7853) --- web/app/components/base/markdown.tsx | 30 ++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/web/app/components/base/markdown.tsx b/web/app/components/base/markdown.tsx index af4b13ff70..6e8ae6c9e6 100644 --- a/web/app/components/base/markdown.tsx +++ b/web/app/components/base/markdown.tsx @@ -8,7 +8,7 @@ import RemarkGfm from 'remark-gfm' import SyntaxHighlighter from 'react-syntax-highlighter' import { atelierHeathLight } from 'react-syntax-highlighter/dist/esm/styles/hljs' import type { RefObject } from 'react' -import { memo, useEffect, useMemo, useRef, useState } from 'react' +import { Component, memo, useEffect, useMemo, useRef, useState } from 'react' import type { CodeComponent } from 'react-markdown/lib/ast-to-react' import cn from '@/utils/classnames' import CopyBtn from '@/app/components/base/copy-btn' @@ -104,7 +104,7 @@ const CodeBlock: CodeComponent = memo(({ inline, className, children, ...props } const match = /language-(\w+)/.exec(className || '') const language = match?.[1] const languageShowName = getCorrectCapitalizationLanguageName(language || '') - let chartData = JSON.parse(String('{"title":{"text":"Something went wrong."}}').replace(/\n$/, '')) + let chartData = JSON.parse(String('{"title":{"text":"ECharts error - Wrong JSON format."}}').replace(/\n$/, '')) if (language === 'echarts') { try { chartData = JSON.parse(String(children).replace(/\n$/, '')) @@ -143,10 +143,10 @@ const CodeBlock: CodeComponent = memo(({ inline, className, children, ...props } ? () : ( (language === 'echarts') - ? (
-
) + ) : ( ) } + +// **Add an ECharts runtime error handler +// Avoid error #7832 (Crash when ECharts accesses undefined objects) +// This can happen when a component attempts to access an undefined object that references an unregistered map, causing the program to crash. + +export default class ErrorBoundary extends Component { + constructor(props) { + super(props) + this.state = { hasError: false } + } + + componentDidCatch(error, errorInfo) { + this.setState({ hasError: true }) + console.error(error, errorInfo) + } + + render() { + if (this.state.hasError) + return
Oops! ECharts reported a runtime error.
(see the browser console for more information)
+ return this.props.children + } +} From 024d688b77f7fafdabf66498ee77efec9631d769 Mon Sep 17 00:00:00 2001 From: Hirotaka Miyagi <31152321+MH4GF@users.noreply.github.com> Date: Sun, 1 Sep 2024 15:57:50 +0900 Subject: [PATCH 20/27] fix(RetrievalConfig): Fix score threshold assignment for zero value (#7865) --- .../nodes/knowledge-retrieval/components/retrieval-config.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx index ebe9d2151e..b335b62e33 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx @@ -70,7 +70,7 @@ const RetrievalConfig: FC = ({ } onMultipleRetrievalConfigChange({ top_k: configs.top_k, - score_threshold: configs.score_threshold_enabled ? (configs.score_threshold || DATASET_DEFAULT.score_threshold) : null, + score_threshold: configs.score_threshold_enabled ? (configs.score_threshold ?? DATASET_DEFAULT.score_threshold) : null, reranking_model: payload.retrieval_mode === RETRIEVE_TYPE.oneWay ? undefined : (!configs.reranking_model?.reranking_provider_name From 1510bdbcf66dfdb14a269eeb22605a0182fdd450 Mon Sep 17 00:00:00 2001 From: Hirotaka Miyagi <31152321+MH4GF@users.noreply.github.com> Date: Sun, 1 Sep 2024 15:58:12 +0900 Subject: [PATCH 21/27] refactor: Remove typecasting by any (#7862) --- web/app/(commonLayout)/apps/Apps.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/(commonLayout)/apps/Apps.tsx b/web/app/(commonLayout)/apps/Apps.tsx index c16512bd50..132096c6b4 100644 --- a/web/app/(commonLayout)/apps/Apps.tsx +++ b/web/app/(commonLayout)/apps/Apps.tsx @@ -139,7 +139,7 @@ const Apps = () => {