diff --git a/api/.env.example b/api/.env.example index 40f35cff6c..00b6c28a91 100644 --- a/api/.env.example +++ b/api/.env.example @@ -383,7 +383,7 @@ LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S LOG_TZ=UTC # Indexing configuration -INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000 +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 # Workflow runtime configuration WORKFLOW_MAX_EXECUTION_STEPS=500 @@ -413,4 +413,4 @@ RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5 CREATE_TIDB_SERVICE_JOB_ENABLED=false -RETRIEVAL_TOP_N=0 \ No newline at end of file +RETRIEVAL_TOP_N=0 diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 275b16a913..e88eb98057 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -647,7 +647,7 @@ class IndexingConfig(BaseSettings): INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field( description="Maximum token length for text segmentation during indexing", - default=1000, + default=4000, ) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index f20261abc2..de3b4f6262 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -106,6 +106,7 @@ class GetProcessRuleApi(Resource): # get default rules mode = DocumentService.DEFAULT_RULES["mode"] rules = DocumentService.DEFAULT_RULES["rules"] + limits = DocumentService.DEFAULT_RULES["limits"] if document_id: # get the latest process rule document = Document.query.get_or_404(document_id) @@ -132,7 +133,7 @@ class GetProcessRuleApi(Resource): mode = dataset_process_rule.mode rules = dataset_process_rule.rules_dict - return {"mode": mode, "rules": rules} + return {"mode": mode, "rules": rules, "limits": limits} class DatasetDocumentListApi(Resource): diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index d38729f31e..a1014e8e0a 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -406,6 +406,9 @@ class DocumentService: ], "segmentation": {"delimiter": "\n", "max_tokens": 500, "chunk_overlap": 50}, }, + "limits": { + "indexing_max_segmentation_tokens_length": dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH, + }, } DOCUMENT_METADATA_SCHEMA = { diff --git a/docker-legacy/docker-compose.yaml b/docker-legacy/docker-compose.yaml index aaca3c9c12..f684c9f1ac 100644 --- a/docker-legacy/docker-compose.yaml +++ b/docker-legacy/docker-compose.yaml @@ -210,7 +210,7 @@ services: SSRF_PROXY_HTTP_URL: 'http://ssrf_proxy:3128' SSRF_PROXY_HTTPS_URL: 'http://ssrf_proxy:3128' # Indexing configuration - INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: 1000 + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: 4000 depends_on: - db - redis diff --git a/docker/.env.example b/docker/.env.example index 3be33f6338..719a025877 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -683,7 +683,7 @@ SMTP_OPPORTUNISTIC_TLS=false # ------------------------------ # Maximum length of segmentation tokens for indexing -INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000 +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 # Member invitation link valid time (hours), # Default: 72. diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index aa3c810d3f..268f77ba24 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -247,7 +247,7 @@ x-shared-env: &shared-api-worker-env SMTP_OPPORTUNISTIC_TLS: ${SMTP_OPPORTUNISTIC_TLS:-false} RESEND_API_KEY: ${RESEND_API_KEY:-your-resend-api-key} RESEND_API_URL: ${RESEND_API_URL:-https://api.resend.com} - INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-1000} + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72} RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5} CODE_EXECUTION_ENDPOINT: ${CODE_EXECUTION_ENDPOINT:-http://sandbox:8194} diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 634f031134..f915c68fef 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -122,7 +122,8 @@ const StepTwo = ({ const setSegmentIdentifier = useCallback((value: string) => { doSetSegmentIdentifier(value ? escape(value) : DEFAULT_SEGMENT_IDENTIFIER) }, []) - const [max, setMax] = useState(4000) // default chunk length + const [maxChunkLength, setMaxChunkLength] = useState(4000) // default chunk length + const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000) const [overlap, setOverlap] = useState(50) const [rules, setRules] = useState([]) const [defaultConfig, setDefaultConfig] = useState() @@ -196,7 +197,7 @@ const StepTwo = ({ const resetRules = () => { if (defaultConfig) { setSegmentIdentifier(defaultConfig.segmentation.separator) - setMax(defaultConfig.segmentation.max_tokens) + setMaxChunkLength(defaultConfig.segmentation.max_tokens) setOverlap(defaultConfig.segmentation.chunk_overlap) setRules(defaultConfig.pre_processing_rules) } @@ -212,8 +213,8 @@ const StepTwo = ({ } const confirmChangeCustomConfig = () => { - if (segmentationType === SegmentType.CUSTOM && max > 4000) { - Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') }) + if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) { + Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) }) return } setCustomFileIndexingEstimate(null) @@ -234,7 +235,7 @@ const StepTwo = ({ pre_processing_rules: rules, segmentation: { separator: unescape(segmentIdentifier), - max_tokens: max, + max_tokens: maxChunkLength, chunk_overlap: overlap, }, } @@ -339,12 +340,12 @@ const StepTwo = ({ ) const getCreationParams = () => { let params - if (segmentationType === SegmentType.CUSTOM && overlap > max) { + if (segmentationType === SegmentType.CUSTOM && overlap > maxChunkLength) { Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.overlapCheck') }) return } - if (segmentationType === SegmentType.CUSTOM && max > 4000) { - Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') }) + if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) { + Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) }) return } if (isSetting) { @@ -415,7 +416,8 @@ const StepTwo = ({ const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' }) const separator = res.rules.segmentation.separator setSegmentIdentifier(separator) - setMax(res.rules.segmentation.max_tokens) + setMaxChunkLength(res.rules.segmentation.max_tokens) + setLimitMaxChunkLength(res.limits.indexing_max_segmentation_tokens_length) setOverlap(res.rules.segmentation.chunk_overlap) setRules(res.rules.pre_processing_rules) setDefaultConfig(res.rules) @@ -432,7 +434,7 @@ const StepTwo = ({ const max = rules.segmentation.max_tokens const overlap = rules.segmentation.chunk_overlap setSegmentIdentifier(separator) - setMax(max) + setMaxChunkLength(max) setOverlap(overlap) setRules(rules.pre_processing_rules) setDefaultConfig(rules) @@ -670,10 +672,10 @@ const StepTwo = ({ type="number" className='h-9' placeholder={t('datasetCreation.stepTwo.maxLength') || ''} - value={max} - max={4000} + value={maxChunkLength} + max={limitMaxChunkLength} min={1} - onChange={e => setMax(parseInt(e.target.value.replace(/^0+/, ''), 10))} + onChange={e => setMaxChunkLength(parseInt(e.target.value.replace(/^0+/, ''), 10))} /> diff --git a/web/i18n/de-DE/dataset-creation.ts b/web/i18n/de-DE/dataset-creation.ts index 18f08814cb..b27a732d18 100644 --- a/web/i18n/de-DE/dataset-creation.ts +++ b/web/i18n/de-DE/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { websiteSource: 'Preprocess-Website', webpageUnit: 'Seiten', separatorTip: 'Ein Trennzeichen ist das Zeichen, das zum Trennen von Text verwendet wird. \\n\\n und \\n sind häufig verwendete Trennzeichen zum Trennen von Absätzen und Zeilen. In Kombination mit Kommas (\\n\\n,\\n) werden Absätze nach Zeilen segmentiert, wenn die maximale Blocklänge überschritten wird. Sie können auch spezielle, von Ihnen selbst definierte Trennzeichen verwenden (z. B. ***).', - maxLengthCheck: 'Die maximale Stücklänge sollte weniger als 4000 betragen', + maxLengthCheck: 'Die maximale Stücklänge sollte weniger als {{limit}} betragen', }, stepThree: { creationTitle: '🎉 Wissen erstellt', diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index de885671a7..6caf0056f8 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -103,7 +103,7 @@ const translation = { separatorTip: 'A delimiter is the character used to separate text. \\n\\n and \\n are commonly used delimiters for separating paragraphs and lines. Combined with commas (\\n\\n,\\n), paragraphs will be segmented by lines when exceeding the maximum chunk length. You can also use special delimiters defined by yourself (e.g. ***).', separatorPlaceholder: '\\n\\n for separating paragraphs; \\n for separating lines', maxLength: 'Maximum chunk length', - maxLengthCheck: 'Maximum chunk length should be less than 4000', + maxLengthCheck: 'Maximum chunk length should be less than {{limit}}', overlap: 'Chunk overlap', overlapTip: 'Setting the chunk overlap can maintain the semantic relevance between them, enhancing the retrieve effect. It is recommended to set 10%-25% of the maximum chunk size.', overlapCheck: 'chunk overlap should not bigger than maximum chunk length', diff --git a/web/i18n/es-ES/dataset-creation.ts b/web/i18n/es-ES/dataset-creation.ts index 02415b4359..2b6cac1b0e 100644 --- a/web/i18n/es-ES/dataset-creation.ts +++ b/web/i18n/es-ES/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: 'Para cambiar el método de índice, por favor ve a la ', datasetSettingLink: 'configuración del conocimiento.', separatorTip: 'Un delimitador es el carácter que se utiliza para separar el texto. \\n\\n y \\n son delimitadores comúnmente utilizados para separar párrafos y líneas. Combinado con comas (\\n\\n,\\n), los párrafos se segmentarán por líneas cuando excedan la longitud máxima del fragmento. También puede utilizar delimitadores especiales definidos por usted mismo (por ejemplo, ***).', - maxLengthCheck: 'La longitud máxima del fragmento debe ser inferior a 4000', + maxLengthCheck: 'La longitud máxima del fragmento debe ser inferior a {{limit}}', }, stepThree: { creationTitle: '🎉 Conocimiento creado', diff --git a/web/i18n/fa-IR/dataset-creation.ts b/web/i18n/fa-IR/dataset-creation.ts index 195d968228..0b5f42827e 100644 --- a/web/i18n/fa-IR/dataset-creation.ts +++ b/web/i18n/fa-IR/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: 'برای تغییر روش شاخص، لطفاً به', datasetSettingLink: 'تنظیمات دانش بروید.', separatorTip: 'جداکننده نویسه ای است که برای جداسازی متن استفاده می شود. \\n\\n و \\n معمولا برای جداسازی پاراگراف ها و خطوط استفاده می شوند. همراه با کاما (\\n\\n,\\n)، پاراگراف ها زمانی که از حداکثر طول تکه فراتر می روند، با خطوط تقسیم بندی می شوند. همچنین می توانید از جداکننده های خاصی که توسط خودتان تعریف شده اند استفاده کنید (مثلا ***).', - maxLengthCheck: 'حداکثر طول تکه باید کمتر از 4000 باشد', + maxLengthCheck: 'حداکثر طول تکه باید کمتر از {{limit}} باشد', }, stepThree: { creationTitle: ' دانش ایجاد شد', diff --git a/web/i18n/fr-FR/dataset-creation.ts b/web/i18n/fr-FR/dataset-creation.ts index 0c866d51e6..c777af4fbe 100644 --- a/web/i18n/fr-FR/dataset-creation.ts +++ b/web/i18n/fr-FR/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { webpageUnit: 'Pages', websiteSource: 'Site web de prétraitement', separatorTip: 'Un délimiteur est le caractère utilisé pour séparer le texte. \\n\\n et \\n sont des délimiteurs couramment utilisés pour séparer les paragraphes et les lignes. Combiné à des virgules (\\n\\n,\\n), les paragraphes seront segmentés par des lignes lorsqu’ils dépasseront la longueur maximale des morceaux. Vous pouvez également utiliser des délimiteurs spéciaux définis par vous-même (par exemple ***).', - maxLengthCheck: 'La longueur maximale des morceaux doit être inférieure à 4000', + maxLengthCheck: 'La longueur maximale des morceaux doit être inférieure à {{limit}}', }, stepThree: { creationTitle: '🎉 Connaissance créée', diff --git a/web/i18n/hi-IN/dataset-creation.ts b/web/i18n/hi-IN/dataset-creation.ts index a242eebb24..51063dcb14 100644 --- a/web/i18n/hi-IN/dataset-creation.ts +++ b/web/i18n/hi-IN/dataset-creation.ts @@ -164,7 +164,7 @@ const translation = { retrievalSettingTip: 'इंडेक्स विधि बदलने के लिए, कृपया जाएं ', datasetSettingLink: 'ज्ञान सेटिंग्स।', separatorTip: 'एक सीमांकक पाठ को अलग करने के लिए उपयोग किया जाने वाला वर्ण है। \\n\\n और \\n आमतौर पर पैराग्राफ और लाइनों को अलग करने के लिए उपयोग किए जाने वाले सीमांकक हैं। अल्पविराम (\\n\\n,\\n) के साथ संयुक्त, अधिकतम खंड लंबाई से अधिक होने पर अनुच्छेदों को पंक्तियों द्वारा खंडित किया जाएगा। आप स्वयं द्वारा परिभाषित विशेष सीमांकक का भी उपयोग कर सकते हैं (उदा. ***).', - maxLengthCheck: 'अधिकतम चंक लंबाई 4000 से कम होनी चाहिए', + maxLengthCheck: 'अधिकतम चंक लंबाई {{limit}} से कम होनी चाहिए', }, stepThree: { creationTitle: '🎉 ज्ञान बनाया गया', diff --git a/web/i18n/it-IT/dataset-creation.ts b/web/i18n/it-IT/dataset-creation.ts index ebbe34f143..af174ce41f 100644 --- a/web/i18n/it-IT/dataset-creation.ts +++ b/web/i18n/it-IT/dataset-creation.ts @@ -167,7 +167,7 @@ const translation = { retrievalSettingTip: 'Per cambiare il metodo di indicizzazione, vai alle ', datasetSettingLink: 'impostazioni della Conoscenza.', separatorTip: 'Un delimitatore è il carattere utilizzato per separare il testo. \\n\\n e \\n sono delimitatori comunemente usati per separare paragrafi e righe. In combinazione con le virgole (\\n\\n,\\n), i paragrafi verranno segmentati per righe quando superano la lunghezza massima del blocco. È inoltre possibile utilizzare delimitatori speciali definiti dall\'utente (ad es. ***).', - maxLengthCheck: 'La lunghezza massima del blocco deve essere inferiore a 4000', + maxLengthCheck: 'La lunghezza massima del blocco deve essere inferiore a {{limit}}', }, stepThree: { creationTitle: '🎉 Conoscenza creata', diff --git a/web/i18n/ja-JP/dataset-creation.ts b/web/i18n/ja-JP/dataset-creation.ts index 597ec5f8f1..9ae6c4e737 100644 --- a/web/i18n/ja-JP/dataset-creation.ts +++ b/web/i18n/ja-JP/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: '検索方法を変更するには、', datasetSettingLink: 'ナレッジ設定', separatorTip: '区切り文字は、テキストを区切るために使用される文字です。\\n\\n と \\n は、段落と行を区切るために一般的に使用される区切り記号です。カンマ (\\n\\n,\\n) と組み合わせると、最大チャンク長を超えると、段落は行で区切られます。自分で定義した特別な区切り文字を使用することもできます(例:***)。', - maxLengthCheck: 'チャンクの最大長は 4000 未満にする必要があります', + maxLengthCheck: 'チャンクの最大長は {{limit}} 未満にする必要があります', }, stepThree: { creationTitle: '🎉 ナレッジが作成されました', diff --git a/web/i18n/ko-KR/dataset-creation.ts b/web/i18n/ko-KR/dataset-creation.ts index 52f7aff75d..ee5abb5189 100644 --- a/web/i18n/ko-KR/dataset-creation.ts +++ b/web/i18n/ko-KR/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { webpageUnit: '페이지', websiteSource: '웹 사이트 전처리', separatorTip: '구분 기호는 텍스트를 구분하는 데 사용되는 문자입니다. \\n\\n 및 \\n은 단락과 줄을 구분하는 데 일반적으로 사용되는 구분 기호입니다. 쉼표(\\n\\n,\\n)와 함께 사용하면 최대 청크 길이를 초과할 경우 단락이 줄로 분할됩니다. 직접 정의한 특수 구분 기호(예: ***)를 사용할 수도 있습니다.', - maxLengthCheck: '최대 청크 길이는 4000 미만이어야 합니다.', + maxLengthCheck: '최대 청크 길이는 {{limit}} 미만이어야 합니다.', }, stepThree: { creationTitle: '🎉 지식이 생성되었습니다', diff --git a/web/i18n/pl-PL/dataset-creation.ts b/web/i18n/pl-PL/dataset-creation.ts index 81adff6336..c12ed66278 100644 --- a/web/i18n/pl-PL/dataset-creation.ts +++ b/web/i18n/pl-PL/dataset-creation.ts @@ -155,7 +155,7 @@ const translation = { webpageUnit: 'Stron', websiteSource: 'Witryna internetowa przetwarzania wstępnego', separatorTip: 'Ogranicznik to znak używany do oddzielania tekstu. \\n\\n i \\n są powszechnie używanymi ogranicznikami do oddzielania akapitów i wierszy. W połączeniu z przecinkami (\\n\\n,\\n), akapity będą segmentowane wierszami po przekroczeniu maksymalnej długości fragmentu. Możesz również skorzystać ze zdefiniowanych przez siebie specjalnych ograniczników (np. ***).', - maxLengthCheck: 'Maksymalna długość porcji powinna być mniejsza niż 4000', + maxLengthCheck: 'Maksymalna długość porcji powinna być mniejsza niż {{limit}}', }, stepThree: { creationTitle: '🎉 Utworzono Wiedzę', diff --git a/web/i18n/pt-BR/dataset-creation.ts b/web/i18n/pt-BR/dataset-creation.ts index 5ab2456562..8534bdce1d 100644 --- a/web/i18n/pt-BR/dataset-creation.ts +++ b/web/i18n/pt-BR/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { websiteSource: 'Site de pré-processamento', webpageUnit: 'Páginas', separatorTip: 'Um delimitador é o caractere usado para separar o texto. \\n\\n e \\n são delimitadores comumente usados para separar parágrafos e linhas. Combinado com vírgulas (\\n\\n,\\n), os parágrafos serão segmentados por linhas ao exceder o comprimento máximo do bloco. Você também pode usar delimitadores especiais definidos por você (por exemplo, ***).', - maxLengthCheck: 'O comprimento máximo do chunk deve ser inferior a 4000', + maxLengthCheck: 'O comprimento máximo do chunk deve ser inferior a {{limit}}', }, stepThree: { creationTitle: '🎉 Conhecimento criado', diff --git a/web/i18n/ro-RO/dataset-creation.ts b/web/i18n/ro-RO/dataset-creation.ts index 2f474f440a..f18b2ac5c2 100644 --- a/web/i18n/ro-RO/dataset-creation.ts +++ b/web/i18n/ro-RO/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { webpageUnit: 'Pagini', websiteSource: 'Site-ul web de preprocesare', separatorTip: 'Un delimitator este caracterul folosit pentru a separa textul. \\n\\n și \\n sunt delimitatori utilizați în mod obișnuit pentru separarea paragrafelor și liniilor. Combinate cu virgule (\\n\\n,\\n), paragrafele vor fi segmentate pe linii atunci când depășesc lungimea maximă a bucății. De asemenea, puteți utiliza delimitatori speciali definiți de dumneavoastră (de exemplu, ***).', - maxLengthCheck: 'Lungimea maximă a bucății trebuie să fie mai mică de 4000', + maxLengthCheck: 'Lungimea maximă a bucății trebuie să fie mai mică de {{limit}}', }, stepThree: { creationTitle: '🎉 Cunoștință creată', diff --git a/web/i18n/ru-RU/dataset-creation.ts b/web/i18n/ru-RU/dataset-creation.ts index 66d92bd2b4..ca88e093bc 100644 --- a/web/i18n/ru-RU/dataset-creation.ts +++ b/web/i18n/ru-RU/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: 'Чтобы изменить метод индексации, пожалуйста, перейдите в ', datasetSettingLink: 'настройки базы знаний.', separatorTip: 'Разделитель — это символ, используемый для разделения текста. \\n\\n и \\n — это часто используемые разделители для разделения абзацев и строк. В сочетании с запятыми (\\n\\n,\\n) абзацы будут сегментированы по строкам, если максимальная длина блока превышает их. Вы также можете использовать специальные разделители, определенные вами (например, ***).', - maxLengthCheck: 'Максимальная длина блока должна быть меньше 4000', + maxLengthCheck: 'Максимальная длина блока должна быть меньше {{limit}}', }, stepThree: { creationTitle: '🎉 База знаний создана', diff --git a/web/i18n/sl-SI/dataset-creation.ts b/web/i18n/sl-SI/dataset-creation.ts index 402066ad40..8dcfa8c3f5 100644 --- a/web/i18n/sl-SI/dataset-creation.ts +++ b/web/i18n/sl-SI/dataset-creation.ts @@ -152,7 +152,7 @@ const translation = { indexSettingTip: 'Če želite spremeniti način indeksiranja in model vdelave, pojdite na ', retrievalSettingTip: 'Če želite spremeniti nastavitve iskanja, pojdite na ', datasetSettingLink: 'nastavitve Znanja.', - maxLengthCheck: 'Največja dolžina kosa mora biti manjša od 4000', + maxLengthCheck: 'Največja dolžina kosa mora biti manjša od {{limit}}', }, stepThree: { creationTitle: '🎉 Znanje ustvarjeno', diff --git a/web/i18n/th-TH/dataset-creation.ts b/web/i18n/th-TH/dataset-creation.ts index 850477c568..16ec1fab3e 100644 --- a/web/i18n/th-TH/dataset-creation.ts +++ b/web/i18n/th-TH/dataset-creation.ts @@ -103,7 +103,7 @@ const translation = { separatorTip: 'ตัวคั่นคืออักขระที่ใช้ในการแยกข้อความ \\n\\n และ \\n เป็นตัวคั่นที่ใช้กันทั่วไปสําหรับการแยกย่อหน้าและบรรทัด เมื่อรวมกับเครื่องหมายจุลภาค (\\n\\n,\\n) ย่อหน้าจะถูกแบ่งตามบรรทัดเมื่อเกินความยาวของก้อนสูงสุด คุณยังสามารถใช้ตัวคั่นพิเศษที่กําหนดโดยตัวคุณเอง (เช่น ***)', separatorPlaceholder: '\\n\\n สําหรับแยกย่อหน้า \\n สําหรับแยกเส้น', maxLength: 'ความยาวก้อนสูงสุด', - maxLengthCheck: 'ความยาวก้อนสูงสุดควรน้อยกว่า 4000', + maxLengthCheck: 'ความยาวก้อนสูงสุดควรน้อยกว่า {{limit}}', overlap: 'การทับซ้อนกันของก้อน', overlapTip: 'การตั้งค่าการทับซ้อนกันของกลุ่มสามารถรักษาความเกี่ยวข้องทางความหมายระหว่างกันได้ ขอแนะนําให้ตั้งค่า 10%-25% ของขนาดก้อนสูงสุด', overlapCheck: 'การทับซ้อนกันของก้อนไม่ควรใหญ่กว่าความยาวของก้อนสูงสุด', diff --git a/web/i18n/tr-TR/dataset-creation.ts b/web/i18n/tr-TR/dataset-creation.ts index 1d5dd19634..11e5789884 100644 --- a/web/i18n/tr-TR/dataset-creation.ts +++ b/web/i18n/tr-TR/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: 'Dizin yöntemini değiştirmek için, lütfen', datasetSettingLink: 'Bilgi ayarlarına gidin.', separatorTip: 'Sınırlayıcı, metni ayırmak için kullanılan karakterdir. \\n\\n ve \\n, paragrafları ve satırları ayırmak için yaygın olarak kullanılan sınırlayıcılardır. Virgüllerle (\\n\\n,\\n) birleştirildiğinde, paragraflar maksimum öbek uzunluğunu aştığında satırlarla bölünür. Kendiniz tarafından tanımlanan özel sınırlayıcıları da kullanabilirsiniz (örn.', - maxLengthCheck: 'Maksimum yığın uzunluğu 4000\'den az olmalıdır', + maxLengthCheck: 'Maksimum yığın uzunluğu {{limit}}\'den az olmalıdır', }, stepThree: { creationTitle: '🎉 Bilgi oluşturuldu', diff --git a/web/i18n/uk-UA/dataset-creation.ts b/web/i18n/uk-UA/dataset-creation.ts index e78934c9e9..d6ee22c2d3 100644 --- a/web/i18n/uk-UA/dataset-creation.ts +++ b/web/i18n/uk-UA/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { webpageUnit: 'Сторінок', websiteSource: 'Веб-сайт попередньої обробки', separatorTip: 'Роздільник – це символ, який використовується для поділу тексту. \\n\\n та \\n є часто використовуваними роздільниками для відокремлення абзаців та рядків. У поєднанні з комами (\\n\\n,\\n) абзаци будуть розділені лініями, якщо вони перевищують максимальну довжину фрагмента. Ви також можете використовувати спеціальні роздільники, визначені вами (наприклад, ***).', - maxLengthCheck: 'Максимальна довжина шматка має бути меншою за 4000', + maxLengthCheck: 'Максимальна довжина шматка має бути меншою за {{limit}}', }, stepThree: { creationTitle: '🎉 Знання створено', diff --git a/web/i18n/vi-VN/dataset-creation.ts b/web/i18n/vi-VN/dataset-creation.ts index 3dc26bee46..045f3d6ea5 100644 --- a/web/i18n/vi-VN/dataset-creation.ts +++ b/web/i18n/vi-VN/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { websiteSource: 'Trang web tiền xử lý', webpageUnit: 'Trang', separatorTip: 'Dấu phân cách là ký tự được sử dụng để phân tách văn bản. \\n\\n và \\n là dấu phân cách thường được sử dụng để tách các đoạn văn và dòng. Kết hợp với dấu phẩy (\\n\\n,\\n), các đoạn văn sẽ được phân đoạn theo các dòng khi vượt quá độ dài đoạn tối đa. Bạn cũng có thể sử dụng dấu phân cách đặc biệt do chính bạn xác định (ví dụ: ***).', - maxLengthCheck: 'Chiều dài đoạn tối đa phải nhỏ hơn 4000', + maxLengthCheck: 'Chiều dài đoạn tối đa phải nhỏ hơn {{limit}}', }, stepThree: { creationTitle: '🎉 Kiến thức đã được tạo', diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index fac809d7e2..e6c3aa00ba 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -103,7 +103,7 @@ const translation = { separatorTip: '分隔符是用于分隔文本的字符。\\n\\n 和 \\n 是常用于分隔段落和行的分隔符。用逗号连接分隔符(\\n\\n,\\n),当段落超过最大块长度时,会按行进行分割。你也可以使用自定义的特殊分隔符(例如 ***)。', separatorPlaceholder: '\\n\\n 用于分段;\\n 用于分行', maxLength: '分段最大长度', - maxLengthCheck: '分段最大长度不能大于 4000', + maxLengthCheck: '分段最大长度不能大于 {{limit}}', overlap: '分段重叠长度', overlapTip: '设置分段之间的重叠长度可以保留分段之间的语义关系,提升召回效果。建议设置为最大分段长度的10%-25%', overlapCheck: '分段重叠长度不能大于分段最大长度', diff --git a/web/i18n/zh-Hant/dataset-creation.ts b/web/i18n/zh-Hant/dataset-creation.ts index 02374573cf..8c5673cb3f 100644 --- a/web/i18n/zh-Hant/dataset-creation.ts +++ b/web/i18n/zh-Hant/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { websiteSource: '預處理網站', webpageUnit: '頁面', separatorTip: '分隔符是用於分隔文字的字元。\\n\\n 和 \\n 是分隔段落和行的常用分隔符。與逗號 (\\n\\n,\\n) 組合使用時,當超過最大區塊長度時,段落將按行分段。您也可以使用自定義的特殊分隔符(例如 ***)。', - maxLengthCheck: '塊最大長度應小於 4000', + maxLengthCheck: '塊最大長度應小於 {{limit}}', }, stepThree: { creationTitle: '🎉 知識庫已建立', diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 95c7ce45de..e2fa608bae 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -175,6 +175,7 @@ export type ProcessMode = 'automatic' | 'custom' export type ProcessRuleResponse = { mode: ProcessMode rules: Rules + limits: Limits } export type Rules = { @@ -182,6 +183,10 @@ export type Rules = { segmentation: Segmentation } +export type Limits = { + indexing_max_segmentation_tokens_length: number +} + export type PreProcessingRule = { id: string enabled: boolean