From fe217da05c3ccbdaf395fc1bd85b7bfbf5d08b3c Mon Sep 17 00:00:00 2001 From: kurokobo Date: Mon, 2 Sep 2024 23:49:56 +0900 Subject: [PATCH 01/22] fix: correct typo in the setting screen (#7897) --- web/i18n/en-US/app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index 1ce517e32b..d8cc2ffec0 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -79,7 +79,7 @@ const translation = { }, answerIcon: { title: 'Use WebApp icon to replace 🤖', - description: 'Wether to use the WebApp icon to replace 🤖 in the shared application', + description: 'Whether to use the WebApp icon to replace 🤖 in the shared application', descriptionInExplore: 'Whether to use the WebApp icon to replace 🤖 in Explore', }, switch: 'Switch to Workflow Orchestrate', From f76bbbf5e6f5f490acf7d6bcee7e5dc5feab7e75 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Tue, 3 Sep 2024 09:48:30 +0800 Subject: [PATCH 02/22] chore(Dockerfile): Bump expat to 2.6.2-2 (#7904) --- api/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/Dockerfile b/api/Dockerfile index 82b89ad77b..ccdec017ab 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -55,7 +55,7 @@ RUN apt-get update \ && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \ && apt-get update \ # For Security - && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-1 libldap-2.5-0=2.5.18+dfsg-3 perl=5.38.2-5 libsqlite3-0=3.46.0-1 \ + && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-2 libldap-2.5-0=2.5.18+dfsg-3 perl=5.38.2-5 libsqlite3-0=3.46.0-1 \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* From 8986be0aab90bdba52e52aae2635ebb4df288212 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Tue, 3 Sep 2024 09:49:32 +0800 Subject: [PATCH 03/22] chore: Update versions to 0.7.3 (#7895) --- api/configs/packaging/__init__.py | 2 +- api/pyproject.toml | 2 -- docker-legacy/docker-compose.yaml | 6 +++--- docker/docker-compose.yaml | 6 +++--- web/package.json | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py index dd09671612..2d540ca584 100644 --- a/api/configs/packaging/__init__.py +++ b/api/configs/packaging/__init__.py @@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings): CURRENT_VERSION: str = Field( description="Dify version", - default="0.7.2", + default="0.7.3", ) COMMIT_SHA: str = Field( diff --git a/api/pyproject.toml b/api/pyproject.toml index f1d5e213ae..6a46ab95f3 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -6,8 +6,6 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.ruff] -exclude = [ -] line-length = 120 [tool.ruff.lint] diff --git a/docker-legacy/docker-compose.yaml b/docker-legacy/docker-compose.yaml index 1f23dc84e0..548a1c004e 100644 --- a/docker-legacy/docker-compose.yaml +++ b/docker-legacy/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3' services: # API service api: - image: langgenius/dify-api:0.7.2 + image: langgenius/dify-api:0.7.3 restart: always environment: # Startup mode, 'api' starts the API server. @@ -229,7 +229,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.7.2 + image: langgenius/dify-api:0.7.3 restart: always environment: CONSOLE_WEB_URL: '' @@ -400,7 +400,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.7.2 + image: langgenius/dify-web:0.7.3 restart: always environment: # The base URL of console application api server, refers to the Console base URL of WEB service if console domain is diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 7a68bb3875..9a532eed07 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -191,7 +191,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:0.7.2 + image: langgenius/dify-api:0.7.3 restart: always environment: # Use the shared environment variables. @@ -211,7 +211,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.7.2 + image: langgenius/dify-api:0.7.3 restart: always environment: # Use the shared environment variables. @@ -230,7 +230,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.7.2 + image: langgenius/dify-web:0.7.3 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} diff --git a/web/package.json b/web/package.json index e896f58886..b72329cc17 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "dify-web", - "version": "0.7.2", + "version": "0.7.3", "private": true, "engines": { "node": ">=18.17.0" From 0cfcc97e9d777cbcbf7929250878074db3f43cb6 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 3 Sep 2024 10:17:05 +0800 Subject: [PATCH 04/22] feat: support auto generate i18n translate (#6964) Co-authored-by: crazywoola <427733928@qq.com> --- .../translate-i18n-base-on-english.yml | 52 +++++ web/i18n/auto-gen-i18n.js | 82 +++++++ web/i18n/{script.js => check-i18n.js} | 1 + web/package.json | 5 +- web/yarn.lock | 210 +++++++++++++++++- 5 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/translate-i18n-base-on-english.yml create mode 100644 web/i18n/auto-gen-i18n.js rename web/i18n/{script.js => check-i18n.js} (98%) diff --git a/.github/workflows/translate-i18n-base-on-english.yml b/.github/workflows/translate-i18n-base-on-english.yml new file mode 100644 index 0000000000..5b63306ef5 --- /dev/null +++ b/.github/workflows/translate-i18n-base-on-english.yml @@ -0,0 +1,52 @@ +name: Check i18n Files and Create PR + +on: + pull_request: + types: [closed] + branches: [main] + +jobs: + check-and-update: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check for file changes in i18n/en-US + id: check_files + run: | + changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'i18n/en-US/*.ts') + echo "Changed files: $changed_files" + if [ -n "$changed_files" ]; then + echo "FILES_CHANGED=true" >> $GITHUB_ENV + else + echo "FILES_CHANGED=false" >> $GITHUB_ENV + fi + + - name: Set up Node.js + if: env.FILES_CHANGED == 'true' + uses: actions/setup-node@v2 + with: + node-version: 'lts/*' + + - name: Install dependencies + if: env.FILES_CHANGED == 'true' + run: yarn install --frozen-lockfile + + - name: Run npm script + if: env.FILES_CHANGED == 'true' + run: npm run auto-gen-i18n + + - name: Create Pull Request + if: env.FILES_CHANGED == 'true' + uses: peter-evans/create-pull-request@v6 + with: + commit-message: Update i18n files based on en-US changes + title: 'chore: translate i18n files' + body: This PR was automatically created to update i18n files based on changes in en-US locale. + branch: chore/automated-i18n-updates diff --git a/web/i18n/auto-gen-i18n.js b/web/i18n/auto-gen-i18n.js new file mode 100644 index 0000000000..c51bc53af3 --- /dev/null +++ b/web/i18n/auto-gen-i18n.js @@ -0,0 +1,82 @@ +/* eslint-disable no-eval */ +const fs = require('node:fs') +const path = require('node:path') +const transpile = require('typescript').transpile +const magicast = require('magicast') +const { parseModule, generateCode, loadFile } = magicast +const bingTranslate = require('bing-translate-api') +const { translate } = bingTranslate +const data = require('./languages.json') + +const targetLanguage = 'en-US' +// https://github.com/plainheart/bing-translate-api/blob/master/src/met/lang.json +const languageKeyMap = data.languages.reduce((map, language) => { + if (language.supported) { + if (language.value === 'zh-Hans' || language.value === 'zh-Hant') + map[language.value] = language.value + else + map[language.value] = language.value.split('-')[0] + } + + return map +}, {}) + +async function translateMissingKeyDeeply(sourceObj, targetObject, toLanguage) { + await Promise.all(Object.keys(sourceObj).map(async (key) => { + if (targetObject[key] === undefined) { + if (typeof sourceObj[key] === 'object') { + targetObject[key] = {} + await translateMissingKeyDeeply(sourceObj[key], targetObject[key], toLanguage) + } + else { + const { translation } = await translate(sourceObj[key], null, languageKeyMap[toLanguage]) + targetObject[key] = translation + // console.log(translation) + } + } + else if (typeof sourceObj[key] === 'object') { + targetObject[key] = targetObject[key] || {} + await translateMissingKeyDeeply(sourceObj[key], targetObject[key], toLanguage) + } + })) +} + +async function autoGenTrans(fileName, toGenLanguage) { + const fullKeyFilePath = path.join(__dirname, targetLanguage, `${fileName}.ts`) + const toGenLanguageFilePath = path.join(__dirname, toGenLanguage, `${fileName}.ts`) + const fullKeyContent = eval(transpile(fs.readFileSync(fullKeyFilePath, 'utf8'))) + // To keep object format and format it for magicast to work: const translation = { ... } => export default {...} + const readContent = await loadFile(toGenLanguageFilePath) + const { code: toGenContent } = generateCode(readContent) + const mod = await parseModule(`export default ${toGenContent.replace('export default translation', '').replace('const translation = ', '')}`) + const toGenOutPut = mod.exports.default + + await translateMissingKeyDeeply(fullKeyContent, toGenOutPut, toGenLanguage) + const { code } = generateCode(mod) + const res = `const translation =${code.replace('export default', '')} + +export default translation +`.replace(/,\n\n/g, ',\n').replace('};', '}') + + fs.writeFileSync(toGenLanguageFilePath, res) +} + +async function main() { + // const fileName = 'workflow' + // Promise.all(Object.keys(languageKeyMap).map(async (toLanguage) => { + // await autoGenTrans(fileName, toLanguage) + // })) + + const files = fs + .readdirSync(path.join(__dirname, targetLanguage)) + .map(file => file.replace(/\.ts/, '')) + .filter(f => f !== 'app-debug') // ast parse error in app-debug + + await Promise.all(files.map(async (file) => { + await Promise.all(Object.keys(languageKeyMap).map(async (language) => { + await autoGenTrans(file, language) + })) + })) +} + +main() diff --git a/web/i18n/script.js b/web/i18n/check-i18n.js similarity index 98% rename from web/i18n/script.js rename to web/i18n/check-i18n.js index 5af8f466ff..fbe1bbd2ef 100644 --- a/web/i18n/script.js +++ b/web/i18n/check-i18n.js @@ -27,6 +27,7 @@ async function getKeysFromLanuage(language) { // console.log(camelCaseFileName) const content = fs.readFileSync(filePath, 'utf8') const translation = eval(transpile(content)) + // console.log(translation) const keys = Object.keys(translation) const nestedKeys = [] const iterateKeys = (obj, prefix = '') => { diff --git a/web/package.json b/web/package.json index b72329cc17..296613b6bb 100644 --- a/web/package.json +++ b/web/package.json @@ -15,7 +15,8 @@ "prepare": "cd ../ && node -e \"if (process.env.NODE_ENV !== 'production'){process.exit(1)} \" || husky install ./web/.husky", "gen-icons": "node ./app/components/base/icons/script.js", "uglify-embed": "node ./bin/uglify-embed", - "check-i18n": "node ./i18n/script.js", + "check-i18n": "node ./i18n/check-i18n.js", + "auto-gen-i18n": "node ./i18n/auto-gen-i18n.js", "test": "jest", "test:watch": "jest --watch" }, @@ -126,6 +127,7 @@ "@types/sortablejs": "^1.15.1", "@types/uuid": "^9.0.8", "autoprefixer": "^10.4.14", + "bing-translate-api": "^4.0.2", "code-inspector-plugin": "^0.13.0", "cross-env": "^7.0.3", "eslint": "^8.36.0", @@ -134,6 +136,7 @@ "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "lint-staged": "^13.2.2", + "magicast": "^0.3.4", "postcss": "^8.4.31", "sass": "^1.61.0", "tailwindcss": "^3.4.4", diff --git a/web/yarn.lock b/web/yarn.lock index d50aa33f3e..007b0ac714 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -406,6 +406,15 @@ "@babel/helper-validator-identifier" "^7.24.7" to-fast-properties "^2.0.0" +"@babel/types@^7.24.0": + version "7.25.2" + resolved "https://registry.npmjs.org/@babel/types/-/types-7.25.2.tgz#55fb231f7dc958cd69ea141a4c2997e819646125" + integrity sha512-YTnYtra7W9e6/oAZEHj0bJehPRUlLH9/fbpT5LfB0NhQXyALCRkRs3zH9v07IYhkgpqX6Z78FnuccZr/l4Fs4Q== + dependencies: + "@babel/helper-string-parser" "^7.24.8" + "@babel/helper-validator-identifier" "^7.24.7" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -1454,6 +1463,11 @@ resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA== +"@sindresorhus/is@^4.0.0": + version "4.6.0" + resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" + integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== + "@sinonjs/commons@^3.0.0": version "3.0.1" resolved "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz#1029357e44ca901a615585f6d27738dbc89084cd" @@ -1481,6 +1495,13 @@ "@swc/counter" "^0.1.3" tslib "^2.4.0" +"@szmarczak/http-timer@^4.0.5": + version "4.0.6" + resolved "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz#b4a914bb62e7c272d4e5989fe4440f812ab1d807" + integrity sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w== + dependencies: + defer-to-connect "^2.0.0" + "@tailwindcss/line-clamp@^0.4.4": version "0.4.4" resolved "https://registry.npmjs.org/@tailwindcss/line-clamp/-/line-clamp-0.4.4.tgz" @@ -1601,6 +1622,16 @@ dependencies: "@babel/types" "^7.20.7" +"@types/cacheable-request@^6.0.1": + version "6.0.3" + resolved "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz#a430b3260466ca7b5ca5bfd735693b36e7a9d183" + integrity sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw== + dependencies: + "@types/http-cache-semantics" "*" + "@types/keyv" "^3.1.4" + "@types/node" "*" + "@types/responselike" "^1.0.0" + "@types/crypto-js@^4.1.1": version "4.1.1" resolved "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz" @@ -1859,6 +1890,11 @@ dependencies: "@types/unist" "*" +"@types/http-cache-semantics@*": + version "4.0.4" + resolved "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz#b979ebad3919799c979b17c72621c0bc0a31c6c4" + integrity sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA== + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.6" resolved "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" @@ -1925,6 +1961,13 @@ resolved "https://registry.npmjs.org/@types/katex/-/katex-0.16.0.tgz" integrity sha512-hz+S3nV6Mym5xPbT9fnO8dDhBFQguMYpY0Ipxv06JMi1ORgnEM4M1ymWDUhUNer3ElLmT583opRo4RzxKmh9jw== +"@types/keyv@^3.1.4": + version "3.1.4" + resolved "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz#3ccdb1c6751b0c7e52300bcdacd5bcbf8faa75b6" + integrity sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg== + dependencies: + "@types/node" "*" + "@types/lodash-es@^4.17.7": version "4.17.7" resolved "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.7.tgz" @@ -2035,6 +2078,13 @@ resolved "https://registry.npmjs.org/@types/recordrtc/-/recordrtc-5.6.11.tgz" integrity sha512-X4XD5nltz0cjmyzsPNegQReOPF+C5ARTfSPAPhqnKV7SsfRta/M4FBJ5AtSInCaEveL71FLLSVQE9mg8Uuo++w== +"@types/responselike@^1.0.0": + version "1.0.3" + resolved "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.3.tgz#cc29706f0a397cfe6df89debfe4bf5cea159db50" + integrity sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw== + dependencies: + "@types/node" "*" + "@types/semver@^7.3.12": version "7.5.0" resolved "https://registry.npmjs.org/@types/semver/-/semver-7.5.0.tgz" @@ -2578,6 +2628,13 @@ binary-extensions@^2.0.0: resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== +bing-translate-api@^4.0.2: + version "4.0.2" + resolved "https://registry.npmjs.org/bing-translate-api/-/bing-translate-api-4.0.2.tgz#52807a128e883bf074b4174c5e674ffca60685e7" + integrity sha512-JJ8XUehnxzOhHU91oy86xEtp8OOMjVEjCZJX042fKxoO19NNvxJ5omeCcxQNFoPbDqVpBJwqiGVquL0oPdQm1Q== + dependencies: + got "^11.8.6" + boolbase@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz" @@ -2670,6 +2727,24 @@ busboy@1.6.0: dependencies: streamsearch "^1.1.0" +cacheable-lookup@^5.0.3: + version "5.0.4" + resolved "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz#5a6b865b2c44357be3d5ebc2a467b032719a7005" + integrity sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA== + +cacheable-request@^7.0.2: + version "7.0.4" + resolved "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.4.tgz#7a33ebf08613178b403635be7b899d3e69bbe817" + integrity sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg== + dependencies: + clone-response "^1.0.2" + get-stream "^5.1.0" + http-cache-semantics "^4.0.0" + keyv "^4.0.0" + lowercase-keys "^2.0.0" + normalize-url "^6.0.1" + responselike "^2.0.0" + call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.4, call-bind@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz" @@ -2893,6 +2968,13 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" +clone-response@^1.0.2: + version "1.0.3" + resolved "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz#af2032aa47816399cf5f0a1d0db902f517abb8c3" + integrity sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA== + dependencies: + mimic-response "^1.0.0" + clsx@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/clsx/-/clsx-2.0.0.tgz" @@ -3464,6 +3546,13 @@ decode-named-character-reference@^1.0.0: dependencies: character-entities "^2.0.0" +decompress-response@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" + integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ== + dependencies: + mimic-response "^3.1.0" + dedent@^1.0.0: version "1.5.3" resolved "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz#99aee19eb9bae55a67327717b6e848d0bf777e5a" @@ -3521,6 +3610,11 @@ default-browser@^4.0.0: execa "^7.1.1" titleize "^3.0.0" +defer-to-connect@^2.0.0: + version "2.0.1" + resolved "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz#8016bdb4143e4632b77a3449c6236277de520587" + integrity sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg== + define-data-property@^1.0.1, define-data-property@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz" @@ -3720,6 +3814,13 @@ emoji-regex@^9.2.2: resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== +end-of-stream@^1.1.0: + version "1.4.4" + resolved "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + enhanced-resolve@^5.12.0: version "5.16.1" resolved "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.16.1.tgz" @@ -4557,6 +4658,13 @@ get-package-type@^0.1.0: resolved "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== +get-stream@^5.1.0: + version "5.2.0" + resolved "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" + integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== + dependencies: + pump "^3.0.0" + get-stream@^6.0.0, get-stream@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz" @@ -4675,6 +4783,23 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" +got@^11.8.6: + version "11.8.6" + resolved "https://registry.npmjs.org/got/-/got-11.8.6.tgz#276e827ead8772eddbcfc97170590b841823233a" + integrity sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g== + dependencies: + "@sindresorhus/is" "^4.0.0" + "@szmarczak/http-timer" "^4.0.5" + "@types/cacheable-request" "^6.0.1" + "@types/responselike" "^1.0.0" + cacheable-lookup "^5.0.3" + cacheable-request "^7.0.2" + decompress-response "^6.0.0" + http2-wrapper "^1.0.0-beta.5.2" + lowercase-keys "^2.0.0" + p-cancelable "^2.0.0" + responselike "^2.0.0" + graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz" @@ -4909,6 +5034,11 @@ htmlparser2@^8.0.1: domutils "^3.0.1" entities "^4.4.0" +http-cache-semantics@^4.0.0: + version "4.1.1" + resolved "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" + integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== + http-proxy-agent@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz#5129800203520d434f142bc78ff3c170800f2b43" @@ -4918,6 +5048,14 @@ http-proxy-agent@^5.0.0: agent-base "6" debug "4" +http2-wrapper@^1.0.0-beta.5.2: + version "1.0.3" + resolved "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz#b8f55e0c1f25d4ebd08b3b0c2c079f9590800b3d" + integrity sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg== + dependencies: + quick-lru "^5.1.1" + resolve-alpn "^1.0.0" + https-proxy-agent@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" @@ -5905,6 +6043,11 @@ jsesc@~0.5.0: resolved "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz" integrity sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA== +json-buffer@3.0.1: + version "3.0.1" + resolved "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" + integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ== + json-parse-even-better-errors@^2.3.0: version "2.3.1" resolved "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz" @@ -5957,6 +6100,13 @@ katex@^0.16.0, katex@^0.16.10: dependencies: commander "^8.3.0" +keyv@^4.0.0: + version "4.5.4" + resolved "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" + integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== + dependencies: + json-buffer "3.0.1" + khroma@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/khroma/-/khroma-2.0.0.tgz" @@ -6128,6 +6278,11 @@ loose-envify@^1.1.0, loose-envify@^1.4.0: dependencies: js-tokens "^3.0.0 || ^4.0.0" +lowercase-keys@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479" + integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA== + lowlight@^1.17.0: version "1.20.0" resolved "https://registry.npmjs.org/lowlight/-/lowlight-1.20.0.tgz" @@ -6160,6 +6315,15 @@ lz-string@^1.5.0: resolved "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz#c1ab50f77887b712621201ba9fd4e3a6ed099941" integrity sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ== +magicast@^0.3.4: + version "0.3.4" + resolved "https://registry.npmjs.org/magicast/-/magicast-0.3.4.tgz#bbda1791d03190a24b00ff3dd18151e7fd381d19" + integrity sha512-TyDF/Pn36bBji9rWKHlZe+PZb6Mx5V8IHCSxk7X4aljM4e/vyDvZZYwHewdVaqiA0nb3ghfHU/6AUpDxWoER2Q== + dependencies: + "@babel/parser" "^7.24.4" + "@babel/types" "^7.24.0" + source-map-js "^1.2.0" + make-dir@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz#c3c2307a771277cd9638305f915c29ae741b614e" @@ -6870,6 +7034,16 @@ mimic-fn@^4.0.0: resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz" integrity sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw== +mimic-response@^1.0.0: + version "1.0.1" + resolved "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" + integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== + +mimic-response@^3.1.0: + version "3.1.0" + resolved "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" + integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== + min-indent@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz" @@ -7010,6 +7184,10 @@ normalize-range@^0.1.2: resolved "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz" integrity sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA== +normalize-url@^6.0.1: + version "6.1.0" + resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" + integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== normalize-wheel@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/normalize-wheel/-/normalize-wheel-1.0.1.tgz#aec886affdb045070d856447df62ecf86146ec45" @@ -7129,7 +7307,7 @@ object.values@^1.1.6, object.values@^1.1.7: define-properties "^1.2.0" es-abstract "^1.22.1" -once@^1.3.0: +once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.npmjs.org/once/-/once-1.4.0.tgz" integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== @@ -7172,6 +7350,11 @@ optionator@^0.9.1: type-check "^0.4.0" word-wrap "^1.2.3" +p-cancelable@^2.0.0: + version "2.1.1" + resolved "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf" + integrity sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg== + p-limit@^2.2.0: version "2.3.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz" @@ -7498,6 +7681,14 @@ psl@^1.1.33: resolved "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== +pump@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" + integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + punycode@^2.1.0: version "2.3.0" resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz" @@ -7535,6 +7726,11 @@ queue-microtask@^1.2.2: resolved "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== +quick-lru@^5.1.1: + version "5.1.1" + resolved "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932" + integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA== + rc-input@~1.3.5: version "1.3.6" resolved "https://registry.npmjs.org/rc-input/-/rc-input-1.3.6.tgz" @@ -7938,6 +8134,11 @@ resize-observer-polyfill@^1.5.1: resolved "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz" integrity sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg== +resolve-alpn@^1.0.0: + version "1.2.1" + resolved "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9" + integrity sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g== + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -7983,6 +8184,13 @@ resolve@^2.0.0-next.4: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" +responselike@^2.0.0: + version "2.0.1" + resolved "https://registry.npmjs.org/responselike/-/responselike-2.0.1.tgz#9a0bc8fdc252f3fb1cca68b016591059ba1422bc" + integrity sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw== + dependencies: + lowercase-keys "^2.0.0" + restore-cursor@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz" From 7fdd9643797b4456b3d976e890968c5e114ad1df Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 3 Sep 2024 14:09:46 +0800 Subject: [PATCH 05/22] fix: frontend handle sometimes server not generate the wrong follow up data struct (#7916) --- web/app/components/base/chat/chat/hooks.ts | 15 ++++++++++----- .../workflow/panel/debug-and-preview/hooks.ts | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/web/app/components/base/chat/chat/hooks.ts b/web/app/components/base/chat/chat/hooks.ts index a70b1c2ed0..4caa8116df 100644 --- a/web/app/components/base/chat/chat/hooks.ts +++ b/web/app/components/base/chat/chat/hooks.ts @@ -372,11 +372,16 @@ export const useChat = ( handleUpdateChatList(newChatList) } if (config?.suggested_questions_after_answer?.enabled && !hasStopResponded.current && onGetSuggestedQuestions) { - const { data }: any = await onGetSuggestedQuestions( - responseItem.id, - newAbortController => suggestedQuestionsAbortControllerRef.current = newAbortController, - ) - setSuggestQuestions(data) + try { + const { data }: any = await onGetSuggestedQuestions( + responseItem.id, + newAbortController => suggestedQuestionsAbortControllerRef.current = newAbortController, + ) + setSuggestQuestions(data) + } + catch (e) { + setSuggestQuestions([]) + } } }, onFile(file) { diff --git a/web/app/components/workflow/panel/debug-and-preview/hooks.ts b/web/app/components/workflow/panel/debug-and-preview/hooks.ts index 155d2a84ac..54d3915a13 100644 --- a/web/app/components/workflow/panel/debug-and-preview/hooks.ts +++ b/web/app/components/workflow/panel/debug-and-preview/hooks.ts @@ -248,11 +248,16 @@ export const useChat = ( } if (config?.suggested_questions_after_answer?.enabled && !hasStopResponded.current && onGetSuggestedQuestions) { - const { data }: any = await onGetSuggestedQuestions( - responseItem.id, - newAbortController => suggestedQuestionsAbortControllerRef.current = newAbortController, - ) - setSuggestQuestions(data) + try { + const { data }: any = await onGetSuggestedQuestions( + responseItem.id, + newAbortController => suggestedQuestionsAbortControllerRef.current = newAbortController, + ) + setSuggestQuestions(data) + } + catch (error) { + setSuggestQuestions([]) + } } }, onMessageEnd: (messageEnd) => { From 01581dd35fbe936874fb1d43efa225d345e80702 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Tue, 3 Sep 2024 17:52:07 +0800 Subject: [PATCH 06/22] improve the notion table extract (#7925) --- api/core/rag/extractor/notion_extractor.py | 25 +++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py index 9535455909..2b12b8a4b2 100644 --- a/api/core/rag/extractor/notion_extractor.py +++ b/api/core/rag/extractor/notion_extractor.py @@ -281,20 +281,25 @@ class NotionExtractor(BaseExtractor): for table_header_cell_text in tabel_header_cell: text = table_header_cell_text["text"]["content"] table_header_cell_texts.append(text) - # get table columns text and format + else: + table_header_cell_texts.append('') + # Initialize Markdown table with headers + markdown_table = "| " + " | ".join(table_header_cell_texts) + " |\n" + markdown_table += "| " + " | ".join(['---'] * len(table_header_cell_texts)) + " |\n" + + # Process data to format each row in Markdown table format results = data["results"] for i in range(len(results) - 1): column_texts = [] - tabel_column_cells = data["results"][i + 1]['table_row']['cells'] - for j in range(len(tabel_column_cells)): - if tabel_column_cells[j]: - for table_column_cell_text in tabel_column_cells[j]: + table_column_cells = data["results"][i + 1]['table_row']['cells'] + for j in range(len(table_column_cells)): + if table_column_cells[j]: + for table_column_cell_text in table_column_cells[j]: column_text = table_column_cell_text["text"]["content"] - column_texts.append(f'{table_header_cell_texts[j]}:{column_text}') - - cur_result_text = "\n".join(column_texts) - result_lines_arr.append(cur_result_text) - + column_texts.append(column_text) + # Add row to Markdown table + markdown_table += "| " + " | ".join(column_texts) + " |\n" + result_lines_arr.append(markdown_table) if data["next_cursor"] is None: done = True break From 1a5116cba084c17ea67252b7d71dec321909a721 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Tue, 3 Sep 2024 18:14:47 +0800 Subject: [PATCH 07/22] Fix/segment create with api (#7928) --- api/controllers/service_api/dataset/segment.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/controllers/service_api/dataset/segment.py b/api/controllers/service_api/dataset/segment.py index 5e10f3b48c..570af339bd 100644 --- a/api/controllers/service_api/dataset/segment.py +++ b/api/controllers/service_api/dataset/segment.py @@ -36,6 +36,10 @@ class SegmentApi(DatasetApiResource): document = DocumentService.get_document(dataset.id, document_id) if not document: raise NotFound("Document not found.") + if document.indexing_status != "completed": + raise NotFound("Document is already completed.") + if not document.enabled: + raise NotFound("Document is disabled.") # check embedding model setting if dataset.indexing_technique == "high_quality": try: From 4b262cae58c4982fe584fe69d5a6533279add614 Mon Sep 17 00:00:00 2001 From: Nam Vu Date: Tue, 3 Sep 2024 18:19:52 +0700 Subject: [PATCH 08/22] chore: #7603 i18n (#7931) --- web/i18n/de-DE/app-overview.ts | 6 +++++- web/i18n/es-ES/app-overview.ts | 6 +++++- web/i18n/fa-IR/app-overview.ts | 6 +++++- web/i18n/fr-FR/app-overview.ts | 6 +++++- web/i18n/hi-IN/app-overview.ts | 7 +++++-- web/i18n/it-IT/app-overview.ts | 7 +++++-- web/i18n/ja-JP/app-overview.ts | 6 +++++- web/i18n/ko-KR/app-overview.ts | 6 +++++- web/i18n/pl-PL/app-overview.ts | 7 +++++-- web/i18n/pt-BR/app-overview.ts | 6 +++++- web/i18n/ro-RO/app-overview.ts | 6 +++++- web/i18n/tr-TR/app-overview.ts | 6 +++++- web/i18n/uk-UA/app-overview.ts | 6 +++++- web/i18n/vi-VN/app-overview.ts | 6 +++++- web/i18n/zh-Hant/app-overview.ts | 6 +++++- 15 files changed, 75 insertions(+), 18 deletions(-) diff --git a/web/i18n/de-DE/app-overview.ts b/web/i18n/de-DE/app-overview.ts index 99100cf868..0304dc6745 100644 --- a/web/i18n/de-DE/app-overview.ts +++ b/web/i18n/de-DE/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'Token/s', totalMessages: { title: 'Gesamtnachrichten', - explanation: 'Tägliche AI-Interaktionszählung; Prompt-Engineering/Debugging ausgenommen.', + explanation: 'Tägliche Anzahl der KI-Interaktionen.', + }, + totalConversations: { + title: 'Gesamte Konversationen', + explanation: 'Tägliche Anzahl der KI-Konversationen; Prompt-Engineering/Debugging ausgeschlossen.', }, activeUsers: { title: 'Aktive Benutzer', diff --git a/web/i18n/es-ES/app-overview.ts b/web/i18n/es-ES/app-overview.ts index f3aaf1f737..98573e8b25 100644 --- a/web/i18n/es-ES/app-overview.ts +++ b/web/i18n/es-ES/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'Token/s', totalMessages: { title: 'Mensajes totales', - explanation: 'Recuento diario de interacciones de IA; excluye la ingeniería/depuración de prompts.', + explanation: 'Recuento diario de interacciones con IA.', + }, + totalConversations: { + title: 'Conversaciones totales', + explanation: 'Recuento diario de conversaciones con IA; ingeniería/depuración de prompts excluida.', }, activeUsers: { title: 'Usuarios activos', diff --git a/web/i18n/fa-IR/app-overview.ts b/web/i18n/fa-IR/app-overview.ts index 1bbd7a0283..fc71b98b75 100644 --- a/web/i18n/fa-IR/app-overview.ts +++ b/web/i18n/fa-IR/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'توکن/ثانیه', totalMessages: { title: 'کل پیام‌ها', - explanation: 'تعداد تعاملات روزانه با AI؛ مهندسی/اشکال‌زدایی دستورات مستثنی هستند.', + explanation: 'تعداد تعاملات روزانه با هوش مصنوعی.', + }, + totalConversations: { + title: 'کل مکالمات', + explanation: 'تعداد مکالمات روزانه با هوش مصنوعی؛ مهندسی/اشکال‌زدایی پرامپت مستثنی است.', }, activeUsers: { title: 'کاربران فعال', diff --git a/web/i18n/fr-FR/app-overview.ts b/web/i18n/fr-FR/app-overview.ts index 23032f9897..5e633a2150 100644 --- a/web/i18n/fr-FR/app-overview.ts +++ b/web/i18n/fr-FR/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'Token/s', totalMessages: { title: 'Total des messages', - explanation: 'Nombre d\'interactions quotidiennes avec l\'IA ; l\'ingénierie/le débogage des prompts sont exclus.', + explanation: 'Nombre d\'interactions quotidiennes avec l\'IA.', + }, + totalConversations: { + title: 'Conversations totales', + explanation: 'Nombre de conversations quotidiennes avec l\'IA ; ingénierie/débogage des prompts exclus.', }, activeUsers: { title: 'Utilisateurs actifs', diff --git a/web/i18n/hi-IN/app-overview.ts b/web/i18n/hi-IN/app-overview.ts index b75206e032..d65920c268 100644 --- a/web/i18n/hi-IN/app-overview.ts +++ b/web/i18n/hi-IN/app-overview.ts @@ -130,8 +130,11 @@ const translation = { tokenPS: 'टोकन/से.', totalMessages: { title: 'कुल संदेश', - explanation: - 'दैनिक एआई इंटरैक्शन की गिनती; प्रॉम्प्ट इंजीनियरिंग/डीबगिंग को शामिल नहीं किया गया।', + explanation: 'दैनिक AI इंटरैक्शन की गिनती।', + }, + totalConversations: { + title: 'कुल वार्तालाप', + explanation: 'दैनिक AI वार्तालाप की गिनती; प्रॉम्प्ट इंजीनियरिंग/डीबगिंग शामिल नहीं।', }, activeUsers: { title: 'सक्रिय उपयोगकर्ता', diff --git a/web/i18n/it-IT/app-overview.ts b/web/i18n/it-IT/app-overview.ts index 380f7e46ad..e7ab5f4080 100644 --- a/web/i18n/it-IT/app-overview.ts +++ b/web/i18n/it-IT/app-overview.ts @@ -132,8 +132,11 @@ const translation = { tokenPS: 'Token/s', totalMessages: { title: 'Totale Messaggi', - explanation: - 'Conteggio delle interazioni giornaliere con l\'AI; ingegneria dei prompt/debug esclusi.', + explanation: 'Conteggio delle interazioni giornaliere con l\'IA.', + }, + totalConversations: { + title: 'Conversazioni totali', + explanation: 'Conteggio delle conversazioni giornaliere con l\'IA; ingegneria/debug dei prompt esclusi.', }, activeUsers: { title: 'Utenti Attivi', diff --git a/web/i18n/ja-JP/app-overview.ts b/web/i18n/ja-JP/app-overview.ts index 1eb6fd1795..720549f578 100644 --- a/web/i18n/ja-JP/app-overview.ts +++ b/web/i18n/ja-JP/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'トークン/秒', totalMessages: { title: 'トータルメッセージ数', - explanation: '日次AIインタラクション数；工学的/デバッグ目的のプロンプトは除外されます。', + explanation: '日次AIインタラクション数。', + }, + totalConversations: { + title: '総会話数', + explanation: '日次AI会話数；プロンプトエンジニアリング/デバッグは除外。', }, activeUsers: { title: 'アクティブユーザー数', diff --git a/web/i18n/ko-KR/app-overview.ts b/web/i18n/ko-KR/app-overview.ts index 47342984b3..8e6c30968e 100644 --- a/web/i18n/ko-KR/app-overview.ts +++ b/web/i18n/ko-KR/app-overview.ts @@ -116,7 +116,11 @@ const translation = { tokenPS: '토큰/초', totalMessages: { title: '총 메시지 수', - explanation: '일일 AI 상호작용 수; 엔지니어링/디버깅 목적의 프롬프트는 제외됩니다.', + explanation: '일일 AI 상호작용 수.', + }, + totalConversations: { + title: '총 대화 수', + explanation: '일일 AI 대화 수; 프롬프트 엔지니어링/디버깅 제외.', }, activeUsers: { title: '활성 사용자 수', diff --git a/web/i18n/pl-PL/app-overview.ts b/web/i18n/pl-PL/app-overview.ts index 95e8aadb70..2b561853e3 100644 --- a/web/i18n/pl-PL/app-overview.ts +++ b/web/i18n/pl-PL/app-overview.ts @@ -130,8 +130,11 @@ const translation = { tokenPS: 'Tokeny/s', totalMessages: { title: 'Łączna liczba wiadomości', - explanation: - 'Dzienna liczba interakcji z AI; inżynieria i debugowanie monitów wykluczone.', + explanation: 'Liczba dziennych interakcji z AI.', + }, + totalConversations: { + title: 'Całkowita liczba rozmów', + explanation: 'Liczba dziennych rozmów z AI; inżynieria/debugowanie promptów wykluczone.', }, activeUsers: { title: 'Aktywni użytkownicy', diff --git a/web/i18n/pt-BR/app-overview.ts b/web/i18n/pt-BR/app-overview.ts index d288e331b3..862a326a42 100644 --- a/web/i18n/pt-BR/app-overview.ts +++ b/web/i18n/pt-BR/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'Token/s', totalMessages: { title: 'Total de Mensagens', - explanation: 'Contagem diária de interações AI; engenharia/de depuração excluída.', + explanation: 'Contagem diária de interações com IA.', + }, + totalConversations: { + title: 'Total de Conversas', + explanation: 'Contagem diária de conversas com IA; engenharia/depuração de prompts excluída.', }, activeUsers: { title: 'Usuários Ativos', diff --git a/web/i18n/ro-RO/app-overview.ts b/web/i18n/ro-RO/app-overview.ts index 007a76a10e..94667b6744 100644 --- a/web/i18n/ro-RO/app-overview.ts +++ b/web/i18n/ro-RO/app-overview.ts @@ -116,7 +116,11 @@ const translation = { tokenPS: 'Token/s', totalMessages: { title: 'Mesaje totale', - explanation: 'Număr de interacțiuni AI zilnice; exclud proiectarea și depanarea promptelor.', + explanation: 'Numărul de interacțiuni zilnice cu IA.', + }, + totalConversations: { + title: 'Total Conversații', + explanation: 'Numărul de conversații zilnice cu IA; ingineria/depanarea prompturilor exclusă.', }, activeUsers: { title: 'Utilizatori activi', diff --git a/web/i18n/tr-TR/app-overview.ts b/web/i18n/tr-TR/app-overview.ts index 77a54dc4b3..f0ee4f0fe4 100644 --- a/web/i18n/tr-TR/app-overview.ts +++ b/web/i18n/tr-TR/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'Token/s', totalMessages: { title: 'Toplam Mesajlar', - explanation: 'Günlük AI etkileşim sayısı; prompt mühendisliği/hata ayıklama hariç.', + explanation: 'Günlük AI etkileşimi sayısı.', + }, + totalConversations: { + title: 'Toplam Konuşmalar', + explanation: 'Günlük AI konuşmaları sayısı; prompt mühendisliği/hata ayıklama hariç.', }, activeUsers: { title: 'Aktif Kullanıcılar', diff --git a/web/i18n/uk-UA/app-overview.ts b/web/i18n/uk-UA/app-overview.ts index 8bd1f0fb39..295773a411 100644 --- a/web/i18n/uk-UA/app-overview.ts +++ b/web/i18n/uk-UA/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'Токени/с', totalMessages: { title: 'Загальна кількість повідомлень', - explanation: 'Щоденна кількість взаємодій з штучним інтелектом; інженерія/налагодження запитів виключено.', + explanation: 'Кількість щоденних взаємодій з ШІ.', + }, + totalConversations: { + title: 'Загальна кількість розмов', + explanation: 'Кількість щоденних розмов з ШІ; інженерія/налагодження промптів виключено.', }, activeUsers: { title: 'Активні користувачі', diff --git a/web/i18n/vi-VN/app-overview.ts b/web/i18n/vi-VN/app-overview.ts index 55a53d73a2..a4e6c96013 100644 --- a/web/i18n/vi-VN/app-overview.ts +++ b/web/i18n/vi-VN/app-overview.ts @@ -119,7 +119,11 @@ const translation = { tokenPS: 'Token/giây', totalMessages: { title: 'Tổng số tin nhắn', - explanation: 'Số lần tương tác AI hàng ngày; không tính việc tạo lại/lặp lại câu hỏi.', + explanation: 'Số lượng tương tác AI hàng ngày.', + }, + totalConversations: { + title: 'Tổng số cuộc hội thoại', + explanation: 'Số lượng cuộc hội thoại AI hàng ngày; không bao gồm kỹ thuật/gỡ lỗi prompt.', }, activeUsers: { title: 'Người dùng hoạt động', diff --git a/web/i18n/zh-Hant/app-overview.ts b/web/i18n/zh-Hant/app-overview.ts index ecf2b5c3b1..df9e80b9a6 100644 --- a/web/i18n/zh-Hant/app-overview.ts +++ b/web/i18n/zh-Hant/app-overview.ts @@ -123,7 +123,11 @@ const translation = { }, activeUsers: { title: '活躍使用者數', - explanation: '與 AI 有效互動，即有一問一答以上的唯一使用者數。提示詞編排和除錯的會話不計入。', + explanation: '每日AI互動次數。', + }, + totalConversations: { + title: '總對話數', + explanation: '每日AI對話次數；不包括提示工程/調試。', }, tokenUsage: { title: '費用消耗', From d8b6c053a2664ead8c72fced06798daab621f0ce Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Tue, 3 Sep 2024 21:25:21 +0800 Subject: [PATCH 09/22] fix rerank model value is empty string (#7937) --- api/core/rag/datasource/retrieval_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index 3932e90042..9f45771794 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -146,7 +146,7 @@ class RetrievalService: ) if documents: - if reranking_model and retrival_method == RetrievalMethod.SEMANTIC_SEARCH.value: + if reranking_model and reranking_model.get('reranking_model_name') and reranking_model.get('reranking_provider_name') and retrival_method == RetrievalMethod.SEMANTIC_SEARCH.value: data_post_processor = DataPostProcessor(str(dataset.tenant_id), RerankMode.RERANKING_MODEL.value, reranking_model, None, False) @@ -180,7 +180,7 @@ class RetrievalService: top_k=top_k ) if documents: - if reranking_model and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH.value: + if reranking_model and reranking_model.get('reranking_model_name') and reranking_model.get('reranking_provider_name') and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH.value: data_post_processor = DataPostProcessor(str(dataset.tenant_id), RerankMode.RERANKING_MODEL.value, reranking_model, None, False) From 596f160a1eb2160b840075a21ad54c6ccf2b2da1 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Wed, 4 Sep 2024 08:32:22 +0800 Subject: [PATCH 10/22] Chore/add default step 1x url (#7933) --- api/core/tools/provider/builtin/stepfun/tools/image.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/core/tools/provider/builtin/stepfun/tools/image.py b/api/core/tools/provider/builtin/stepfun/tools/image.py index 5e544aada6..c571f54675 100644 --- a/api/core/tools/provider/builtin/stepfun/tools/image.py +++ b/api/core/tools/provider/builtin/stepfun/tools/image.py @@ -17,11 +17,8 @@ class StepfunTool(BuiltinTool): """ invoke tools """ - base_url = self.runtime.credentials.get('stepfun_base_url', None) - if not base_url: - base_url = None - else: - base_url = str(URL(base_url) / 'v1') + base_url = self.runtime.credentials.get('stepfun_base_url', 'https://api.stepfun.com') + base_url = str(URL(base_url) / 'v1') client = OpenAI( api_key=self.runtime.credentials['stepfun_api_key'], From c5bdf085581f7212651c5a8c7e449aaea7aa0c14 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Wed, 4 Sep 2024 08:33:02 +0800 Subject: [PATCH 11/22] Chore/add roadmap (#7943) --- web/app/components/header/account-dropdown/index.tsx | 11 ++++++++++- web/i18n/de-DE/common.ts | 2 +- web/i18n/en-US/common.ts | 3 ++- web/i18n/es-ES/common.ts | 2 +- web/i18n/fa-IR/common.ts | 2 +- web/i18n/fr-FR/common.ts | 2 +- web/i18n/hi-IN/common.ts | 2 +- web/i18n/it-IT/common.ts | 2 +- web/i18n/ja-JP/common.ts | 2 +- web/i18n/ko-KR/common.ts | 2 +- web/i18n/pl-PL/common.ts | 2 +- web/i18n/pt-BR/common.ts | 2 +- web/i18n/ro-RO/common.ts | 2 +- web/i18n/ru-RU/common.ts | 2 +- web/i18n/tr-TR/common.ts | 2 +- web/i18n/uk-UA/common.ts | 2 +- web/i18n/vi-VN/common.ts | 2 +- web/i18n/zh-Hans/common.ts | 3 ++- web/i18n/zh-Hant/common.ts | 2 +- 19 files changed, 30 insertions(+), 19 deletions(-) diff --git a/web/app/components/header/account-dropdown/index.tsx b/web/app/components/header/account-dropdown/index.tsx index 2298bff82d..1dd03e5290 100644 --- a/web/app/components/header/account-dropdown/index.tsx +++ b/web/app/components/header/account-dropdown/index.tsx @@ -125,7 +125,7 @@ export default function AppSelector({ isMobile }: IAppSelecotr) { className={classNames(itemClassName, 'group justify-between')} href='https://github.com/langgenius/dify/discussions/categories/feedbacks' target='_blank' rel='noopener noreferrer'> -
{t('common.userProfile.roadmapAndFeedback')}
+
{t('common.userProfile.communityFeedback')}
@@ -149,6 +149,15 @@ export default function AppSelector({ isMobile }: IAppSelecotr) { + + +
{t('common.userProfile.roadmap')}
+ + + { document?.body?.getAttribute('data-public-site-about') !== 'hide' && ( diff --git a/web/i18n/de-DE/common.ts b/web/i18n/de-DE/common.ts index 9a66d5b175..7c3d014b31 100644 --- a/web/i18n/de-DE/common.ts +++ b/web/i18n/de-DE/common.ts @@ -128,7 +128,7 @@ const translation = { workspace: 'Arbeitsbereich', createWorkspace: 'Arbeitsbereich erstellen', helpCenter: 'Hilfe', - roadmapAndFeedback: 'Feedback', + communityFeedback: 'Feedback', community: 'Gemeinschaft', about: 'Über', logout: 'Abmelden', diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index 87dab5cb71..85ac1c0654 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -132,7 +132,8 @@ const translation = { workspace: 'Workspace', createWorkspace: 'Create Workspace', helpCenter: 'Help', - roadmapAndFeedback: 'Feedback', + communityFeedback: 'Feedback', + roadmap: 'Roadmap', community: 'Community', about: 'About', logout: 'Log out', diff --git a/web/i18n/es-ES/common.ts b/web/i18n/es-ES/common.ts index fc37775263..35131b0ba6 100644 --- a/web/i18n/es-ES/common.ts +++ b/web/i18n/es-ES/common.ts @@ -132,7 +132,7 @@ const translation = { workspace: 'Espacio de trabajo', createWorkspace: 'Crear espacio de trabajo', helpCenter: 'Ayuda', - roadmapAndFeedback: 'Comentarios', + communityFeedback: 'Comentarios', community: 'Comunidad', about: 'Acerca de', logout: 'Cerrar sesión', diff --git a/web/i18n/fa-IR/common.ts b/web/i18n/fa-IR/common.ts index e4417bcbcc..02c9c6cf06 100644 --- a/web/i18n/fa-IR/common.ts +++ b/web/i18n/fa-IR/common.ts @@ -132,7 +132,7 @@ const translation = { workspace: 'فضای کاری', createWorkspace: 'ایجاد فضای کاری', helpCenter: 'راهنما', - roadmapAndFeedback: 'بازخورد', + communityFeedback: 'بازخورد', community: 'انجمن', about: 'درباره', logout: 'خروج', diff --git a/web/i18n/fr-FR/common.ts b/web/i18n/fr-FR/common.ts index 2ae0731006..14df722566 100644 --- a/web/i18n/fr-FR/common.ts +++ b/web/i18n/fr-FR/common.ts @@ -128,7 +128,7 @@ const translation = { workspace: 'Espace de travail', createWorkspace: 'Créer un Espace de Travail', helpCenter: 'Aide', - roadmapAndFeedback: 'Retour d\'information', + communityFeedback: 'Retour d\'information', community: 'Communauté', about: 'À propos', logout: 'Se déconnecter', diff --git a/web/i18n/hi-IN/common.ts b/web/i18n/hi-IN/common.ts index 0a210072e1..68aed6bec5 100644 --- a/web/i18n/hi-IN/common.ts +++ b/web/i18n/hi-IN/common.ts @@ -137,7 +137,7 @@ const translation = { workspace: 'वर्कस्पेस', createWorkspace: 'वर्कस्पेस बनाएं', helpCenter: 'सहायता', - roadmapAndFeedback: 'प्रतिक्रिया', + communityFeedback: 'प्रतिक्रिया', community: 'समुदाय', about: 'के बारे में', logout: 'लॉग आउट', diff --git a/web/i18n/it-IT/common.ts b/web/i18n/it-IT/common.ts index 595a5075eb..ad3fb1fb92 100644 --- a/web/i18n/it-IT/common.ts +++ b/web/i18n/it-IT/common.ts @@ -137,7 +137,7 @@ const translation = { workspace: 'Workspace', createWorkspace: 'Crea Workspace', helpCenter: 'Aiuto', - roadmapAndFeedback: 'Feedback', + communityFeedback: 'Feedback', community: 'Comunità', about: 'Informazioni', logout: 'Esci', diff --git a/web/i18n/ja-JP/common.ts b/web/i18n/ja-JP/common.ts index fc61141bd3..7a3a7b0550 100644 --- a/web/i18n/ja-JP/common.ts +++ b/web/i18n/ja-JP/common.ts @@ -132,7 +132,7 @@ const translation = { workspace: 'ワークスペース', createWorkspace: 'ワークスペースを作成', helpCenter: 'ヘルプ', - roadmapAndFeedback: 'フィードバック', + communityFeedback: 'フィードバック', community: 'コミュニティ', about: 'Difyについて', logout: 'ログアウト', diff --git a/web/i18n/ko-KR/common.ts b/web/i18n/ko-KR/common.ts index edd0295b89..52a4d6beab 100644 --- a/web/i18n/ko-KR/common.ts +++ b/web/i18n/ko-KR/common.ts @@ -124,7 +124,7 @@ const translation = { workspace: '작업 공간', createWorkspace: '작업 공간 만들기', helpCenter: '도움말 센터', - roadmapAndFeedback: '로드맵 및 피드백', + communityFeedback: '로드맵 및 피드백', community: '커뮤니티', about: 'Dify 소개', logout: '로그아웃', diff --git a/web/i18n/pl-PL/common.ts b/web/i18n/pl-PL/common.ts index 1f41abe154..856859539b 100644 --- a/web/i18n/pl-PL/common.ts +++ b/web/i18n/pl-PL/common.ts @@ -133,7 +133,7 @@ const translation = { workspace: 'Przestrzeń robocza', createWorkspace: 'Utwórz przestrzeń roboczą', helpCenter: 'Pomoc', - roadmapAndFeedback: 'Opinie', + communityFeedback: 'Opinie', community: 'Społeczność', about: 'O', logout: 'Wyloguj się', diff --git a/web/i18n/pt-BR/common.ts b/web/i18n/pt-BR/common.ts index f93979404b..dde7007bd6 100644 --- a/web/i18n/pt-BR/common.ts +++ b/web/i18n/pt-BR/common.ts @@ -128,7 +128,7 @@ const translation = { workspace: 'Espaço de trabalho', createWorkspace: 'Criar Espaço de Trabalho', helpCenter: 'Ajuda', - roadmapAndFeedback: 'Feedback', + communityFeedback: 'Feedback', community: 'Comunidade', about: 'Sobre', logout: 'Sair', diff --git a/web/i18n/ro-RO/common.ts b/web/i18n/ro-RO/common.ts index 34ca1c4671..7c3fbdb7f2 100644 --- a/web/i18n/ro-RO/common.ts +++ b/web/i18n/ro-RO/common.ts @@ -127,7 +127,7 @@ const translation = { workspace: 'Spațiu de lucru', createWorkspace: 'Creează Spațiu de lucru', helpCenter: 'Ajutor', - roadmapAndFeedback: 'Feedback', + communityFeedback: 'Feedback', community: 'Comunitate', about: 'Despre', logout: 'Deconectare', diff --git a/web/i18n/ru-RU/common.ts b/web/i18n/ru-RU/common.ts index 6aa98c9a1e..69cd159c34 100644 --- a/web/i18n/ru-RU/common.ts +++ b/web/i18n/ru-RU/common.ts @@ -132,7 +132,7 @@ const translation = { workspace: 'Рабочее пространство', createWorkspace: 'Создать рабочее пространство', helpCenter: 'Помощь', - roadmapAndFeedback: 'Обратная связь', + communityFeedback: 'Обратная связь', community: 'Сообщество', about: 'О нас', logout: 'Выйти', diff --git a/web/i18n/tr-TR/common.ts b/web/i18n/tr-TR/common.ts index a194ffd769..df7fc1bccb 100644 --- a/web/i18n/tr-TR/common.ts +++ b/web/i18n/tr-TR/common.ts @@ -132,7 +132,7 @@ const translation = { workspace: 'Çalışma Alanı', createWorkspace: 'Çalışma Alanı Oluştur', helpCenter: 'Yardım', - roadmapAndFeedback: 'Geri Bildirim', + communityFeedback: 'Geri Bildirim', community: 'Topluluk', about: 'Hakkında', logout: 'Çıkış Yap', diff --git a/web/i18n/uk-UA/common.ts b/web/i18n/uk-UA/common.ts index 33324ce0f2..5e7dc1dab8 100644 --- a/web/i18n/uk-UA/common.ts +++ b/web/i18n/uk-UA/common.ts @@ -128,7 +128,7 @@ const translation = { workspace: 'Робочий простір', createWorkspace: 'Створити робочий простір', helpCenter: 'Довідковий центр', - roadmapAndFeedback: 'відгуки', + communityFeedback: 'відгуки', community: 'Спільнота', about: 'Про нас', logout: 'Вийти', diff --git a/web/i18n/vi-VN/common.ts b/web/i18n/vi-VN/common.ts index 19855d31f0..6eb142fca3 100644 --- a/web/i18n/vi-VN/common.ts +++ b/web/i18n/vi-VN/common.ts @@ -128,7 +128,7 @@ const translation = { workspace: 'Không gian làm việc', createWorkspace: 'Tạo Không gian làm việc', helpCenter: 'Trung tâm trợ giúp', - roadmapAndFeedback: 'Phản hồi', + communityFeedback: 'Phản hồi', community: 'Cộng đồng', about: 'Về chúng tôi', logout: 'Đăng xuất', diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index 5333d18763..477dd363c2 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -132,7 +132,8 @@ const translation = { workspace: '工作空间', createWorkspace: '创建工作空间', helpCenter: '帮助文档', - roadmapAndFeedback: '用户反馈', + communityFeedback: '用户反馈', + roadmap: '路线图', community: '社区', about: '关于', logout: '登出', diff --git a/web/i18n/zh-Hant/common.ts b/web/i18n/zh-Hant/common.ts index 02296907d9..d4c21e992b 100644 --- a/web/i18n/zh-Hant/common.ts +++ b/web/i18n/zh-Hant/common.ts @@ -128,7 +128,7 @@ const translation = { workspace: '工作空間', createWorkspace: '建立工作空間', helpCenter: '幫助文件', - roadmapAndFeedback: '使用者反饋', + communityFeedback: '使用者反饋', community: '社群', about: '關於', logout: '登出', From 80d14c9b225559cc208e92e1854d4bc0ff1f6aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=B3=E9=88=9E?= Date: Wed, 4 Sep 2024 08:41:12 +0800 Subject: [PATCH 12/22] fix(api): Code-Based Extension cause error on position map sorting (#7934) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 陳鈞 --- api/core/extension/extensible.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/extension/extensible.py b/api/core/extension/extensible.py index 0296126d8b..8d73aa2b8b 100644 --- a/api/core/extension/extensible.py +++ b/api/core/extension/extensible.py @@ -65,7 +65,7 @@ class Extensible: if os.path.exists(builtin_file_path): with open(builtin_file_path, encoding='utf-8') as f: position = int(f.read().strip()) - position_map[extension_name] = position + position_map[extension_name] = position if (extension_name + '.py') not in file_names: logging.warning(f"Missing {extension_name}.py file in {subdir_path}, Skip.") From 71bff9fcf3ae47ba412f023fb0339c5011529a26 Mon Sep 17 00:00:00 2001 From: Nam Vu Date: Wed, 4 Sep 2024 08:42:25 +0700 Subject: [PATCH 13/22] chore: #7943 i18n (#7948) --- web/i18n/de-DE/common.ts | 3 ++- web/i18n/es-ES/common.ts | 1 + web/i18n/fa-IR/common.ts | 1 + web/i18n/fr-FR/common.ts | 1 + web/i18n/hi-IN/common.ts | 1 + web/i18n/it-IT/common.ts | 1 + web/i18n/ja-JP/common.ts | 1 + web/i18n/ko-KR/common.ts | 1 + web/i18n/pl-PL/common.ts | 1 + web/i18n/pt-BR/common.ts | 1 + web/i18n/ro-RO/common.ts | 1 + web/i18n/ru-RU/common.ts | 1 + web/i18n/tr-TR/common.ts | 1 + web/i18n/uk-UA/common.ts | 1 + web/i18n/vi-VN/common.ts | 1 + web/i18n/zh-Hant/common.ts | 1 + 16 files changed, 17 insertions(+), 1 deletion(-) diff --git a/web/i18n/de-DE/common.ts b/web/i18n/de-DE/common.ts index 7c3d014b31..c9f9e9fc22 100644 --- a/web/i18n/de-DE/common.ts +++ b/web/i18n/de-DE/common.ts @@ -128,7 +128,8 @@ const translation = { workspace: 'Arbeitsbereich', createWorkspace: 'Arbeitsbereich erstellen', helpCenter: 'Hilfe', - communityFeedback: 'Feedback', + communityFeedback: 'Rückmeldung', + roadmap: 'Fahrplan', community: 'Gemeinschaft', about: 'Über', logout: 'Abmelden', diff --git a/web/i18n/es-ES/common.ts b/web/i18n/es-ES/common.ts index 35131b0ba6..119f465f76 100644 --- a/web/i18n/es-ES/common.ts +++ b/web/i18n/es-ES/common.ts @@ -133,6 +133,7 @@ const translation = { createWorkspace: 'Crear espacio de trabajo', helpCenter: 'Ayuda', communityFeedback: 'Comentarios', + roadmap: 'Hoja de ruta', community: 'Comunidad', about: 'Acerca de', logout: 'Cerrar sesión', diff --git a/web/i18n/fa-IR/common.ts b/web/i18n/fa-IR/common.ts index 02c9c6cf06..42c6865a6e 100644 --- a/web/i18n/fa-IR/common.ts +++ b/web/i18n/fa-IR/common.ts @@ -133,6 +133,7 @@ const translation = { createWorkspace: 'ایجاد فضای کاری', helpCenter: 'راهنما', communityFeedback: 'بازخورد', + roadmap: 'نقشه راه', community: 'انجمن', about: 'درباره', logout: 'خروج', diff --git a/web/i18n/fr-FR/common.ts b/web/i18n/fr-FR/common.ts index 14df722566..f0699eca7a 100644 --- a/web/i18n/fr-FR/common.ts +++ b/web/i18n/fr-FR/common.ts @@ -129,6 +129,7 @@ const translation = { createWorkspace: 'Créer un Espace de Travail', helpCenter: 'Aide', communityFeedback: 'Retour d\'information', + roadmap: 'Feuille de route', community: 'Communauté', about: 'À propos', logout: 'Se déconnecter', diff --git a/web/i18n/hi-IN/common.ts b/web/i18n/hi-IN/common.ts index 68aed6bec5..3316397cb6 100644 --- a/web/i18n/hi-IN/common.ts +++ b/web/i18n/hi-IN/common.ts @@ -138,6 +138,7 @@ const translation = { createWorkspace: 'वर्कस्पेस बनाएं', helpCenter: 'सहायता', communityFeedback: 'प्रतिक्रिया', + roadmap: 'रोडमैप', community: 'समुदाय', about: 'के बारे में', logout: 'लॉग आउट', diff --git a/web/i18n/it-IT/common.ts b/web/i18n/it-IT/common.ts index ad3fb1fb92..01560b9374 100644 --- a/web/i18n/it-IT/common.ts +++ b/web/i18n/it-IT/common.ts @@ -138,6 +138,7 @@ const translation = { createWorkspace: 'Crea Workspace', helpCenter: 'Aiuto', communityFeedback: 'Feedback', + roadmap: 'Tabella di marcia', community: 'Comunità', about: 'Informazioni', logout: 'Esci', diff --git a/web/i18n/ja-JP/common.ts b/web/i18n/ja-JP/common.ts index 7a3a7b0550..4d756d4d7b 100644 --- a/web/i18n/ja-JP/common.ts +++ b/web/i18n/ja-JP/common.ts @@ -133,6 +133,7 @@ const translation = { createWorkspace: 'ワークスペースを作成', helpCenter: 'ヘルプ', communityFeedback: 'フィードバック', + roadmap: 'ロードマップ', community: 'コミュニティ', about: 'Difyについて', logout: 'ログアウト', diff --git a/web/i18n/ko-KR/common.ts b/web/i18n/ko-KR/common.ts index 52a4d6beab..a4e77abb81 100644 --- a/web/i18n/ko-KR/common.ts +++ b/web/i18n/ko-KR/common.ts @@ -125,6 +125,7 @@ const translation = { createWorkspace: '작업 공간 만들기', helpCenter: '도움말 센터', communityFeedback: '로드맵 및 피드백', + roadmap: '로드맵', community: '커뮤니티', about: 'Dify 소개', logout: '로그아웃', diff --git a/web/i18n/pl-PL/common.ts b/web/i18n/pl-PL/common.ts index 856859539b..5125e849ac 100644 --- a/web/i18n/pl-PL/common.ts +++ b/web/i18n/pl-PL/common.ts @@ -134,6 +134,7 @@ const translation = { createWorkspace: 'Utwórz przestrzeń roboczą', helpCenter: 'Pomoc', communityFeedback: 'Opinie', + roadmap: 'Plan działania', community: 'Społeczność', about: 'O', logout: 'Wyloguj się', diff --git a/web/i18n/pt-BR/common.ts b/web/i18n/pt-BR/common.ts index dde7007bd6..04b8c3b49c 100644 --- a/web/i18n/pt-BR/common.ts +++ b/web/i18n/pt-BR/common.ts @@ -129,6 +129,7 @@ const translation = { createWorkspace: 'Criar Espaço de Trabalho', helpCenter: 'Ajuda', communityFeedback: 'Feedback', + roadmap: 'Roteiro', community: 'Comunidade', about: 'Sobre', logout: 'Sair', diff --git a/web/i18n/ro-RO/common.ts b/web/i18n/ro-RO/common.ts index 7c3fbdb7f2..35948d3657 100644 --- a/web/i18n/ro-RO/common.ts +++ b/web/i18n/ro-RO/common.ts @@ -128,6 +128,7 @@ const translation = { createWorkspace: 'Creează Spațiu de lucru', helpCenter: 'Ajutor', communityFeedback: 'Feedback', + roadmap: 'Plan de acțiune', community: 'Comunitate', about: 'Despre', logout: 'Deconectare', diff --git a/web/i18n/ru-RU/common.ts b/web/i18n/ru-RU/common.ts index 69cd159c34..25a3d1031a 100644 --- a/web/i18n/ru-RU/common.ts +++ b/web/i18n/ru-RU/common.ts @@ -133,6 +133,7 @@ const translation = { createWorkspace: 'Создать рабочее пространство', helpCenter: 'Помощь', communityFeedback: 'Обратная связь', + roadmap: 'План развития', community: 'Сообщество', about: 'О нас', logout: 'Выйти', diff --git a/web/i18n/tr-TR/common.ts b/web/i18n/tr-TR/common.ts index df7fc1bccb..b0492b557a 100644 --- a/web/i18n/tr-TR/common.ts +++ b/web/i18n/tr-TR/common.ts @@ -133,6 +133,7 @@ const translation = { createWorkspace: 'Çalışma Alanı Oluştur', helpCenter: 'Yardım', communityFeedback: 'Geri Bildirim', + roadmap: 'Yol haritası', community: 'Topluluk', about: 'Hakkında', logout: 'Çıkış Yap', diff --git a/web/i18n/uk-UA/common.ts b/web/i18n/uk-UA/common.ts index 5e7dc1dab8..644df7d62a 100644 --- a/web/i18n/uk-UA/common.ts +++ b/web/i18n/uk-UA/common.ts @@ -129,6 +129,7 @@ const translation = { createWorkspace: 'Створити робочий простір', helpCenter: 'Довідковий центр', communityFeedback: 'відгуки', + roadmap: 'Дорожня карта', community: 'Спільнота', about: 'Про нас', logout: 'Вийти', diff --git a/web/i18n/vi-VN/common.ts b/web/i18n/vi-VN/common.ts index 6eb142fca3..5c3d7410de 100644 --- a/web/i18n/vi-VN/common.ts +++ b/web/i18n/vi-VN/common.ts @@ -129,6 +129,7 @@ const translation = { createWorkspace: 'Tạo Không gian làm việc', helpCenter: 'Trung tâm trợ giúp', communityFeedback: 'Phản hồi', + roadmap: 'Lộ trình', community: 'Cộng đồng', about: 'Về chúng tôi', logout: 'Đăng xuất', diff --git a/web/i18n/zh-Hant/common.ts b/web/i18n/zh-Hant/common.ts index d4c21e992b..a5bcb4c72f 100644 --- a/web/i18n/zh-Hant/common.ts +++ b/web/i18n/zh-Hant/common.ts @@ -129,6 +129,7 @@ const translation = { createWorkspace: '建立工作空間', helpCenter: '幫助文件', communityFeedback: '使用者反饋', + roadmap: '路線圖', community: '社群', about: '關於', logout: '登出', From 161cc0cda9d2a6128f5ab128eb0d263de2d8b032 Mon Sep 17 00:00:00 2001 From: takatost Date: Wed, 4 Sep 2024 10:00:55 +0800 Subject: [PATCH 14/22] Revert "fix: an issue of keyword search feature in application log list" (#7949) --- api/controllers/console/app/conversation.py | 27 ++++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/api/controllers/console/app/conversation.py b/api/controllers/console/app/conversation.py index c3aac6690e..753a6be20c 100644 --- a/api/controllers/console/app/conversation.py +++ b/api/controllers/console/app/conversation.py @@ -173,18 +173,21 @@ class ChatConversationApi(Resource): if args["keyword"]: keyword_filter = "%{}%".format(args["keyword"]) - message_subquery = ( - db.session.query(Message.conversation_id) - .filter(or_(Message.query.ilike(keyword_filter), Message.answer.ilike(keyword_filter))) - .subquery() - ) - query = query.join(subquery, subquery.c.conversation_id == Conversation.id).filter( - or_( - Conversation.id.in_(message_subquery), - Conversation.name.ilike(keyword_filter), - Conversation.introduction.ilike(keyword_filter), - subquery.c.from_end_user_session_id.ilike(keyword_filter), - ), + query = ( + query.join( + Message, + Message.conversation_id == Conversation.id, + ) + .join(subquery, subquery.c.conversation_id == Conversation.id) + .filter( + or_( + Message.query.ilike(keyword_filter), + Message.answer.ilike(keyword_filter), + Conversation.name.ilike(keyword_filter), + Conversation.introduction.ilike(keyword_filter), + subquery.c.from_end_user_session_id.ilike(keyword_filter), + ), + ) ) account = current_user From fee4d3f6ca818bf150278fb6771c95bc6a257cbe Mon Sep 17 00:00:00 2001 From: Joe <79627742+ZhouhaoJiang@users.noreply.github.com> Date: Wed, 4 Sep 2024 10:39:00 +0800 Subject: [PATCH 15/22] feat: ops trace add llm model (#7306) --- api/core/ops/langfuse_trace/langfuse_trace.py | 1 + api/core/ops/langsmith_trace/langsmith_trace.py | 9 +++++++-- api/core/workflow/nodes/llm/llm_node.py | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/api/core/ops/langfuse_trace/langfuse_trace.py b/api/core/ops/langfuse_trace/langfuse_trace.py index a21c67ed50..a0f3ac7f86 100644 --- a/api/core/ops/langfuse_trace/langfuse_trace.py +++ b/api/core/ops/langfuse_trace/langfuse_trace.py @@ -204,6 +204,7 @@ class LangFuseDataTrace(BaseTraceInstance): node_generation_data = LangfuseGeneration( name="llm", trace_id=trace_id, + model=process_data.get("model_name"), parent_observation_id=node_execution_id, start_time=created_at, end_time=finished_at, diff --git a/api/core/ops/langsmith_trace/langsmith_trace.py b/api/core/ops/langsmith_trace/langsmith_trace.py index fde8a06c61..cc242905bd 100644 --- a/api/core/ops/langsmith_trace/langsmith_trace.py +++ b/api/core/ops/langsmith_trace/langsmith_trace.py @@ -139,8 +139,7 @@ class LangSmithDataTrace(BaseTraceInstance): json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} ) node_total_tokens = execution_metadata.get("total_tokens", 0) - - metadata = json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} + metadata = execution_metadata.copy() metadata.update( { "workflow_run_id": trace_info.workflow_run_id, @@ -156,6 +155,12 @@ class LangSmithDataTrace(BaseTraceInstance): process_data = json.loads(node_execution.process_data) if node_execution.process_data else {} if process_data and process_data.get("model_mode") == "chat": run_type = LangSmithRunType.llm + metadata.update( + { + 'ls_provider': process_data.get('model_provider', ''), + 'ls_model_name': process_data.get('model_name', ''), + } + ) elif node_type == "knowledge-retrieval": run_type = LangSmithRunType.retriever else: diff --git a/api/core/workflow/nodes/llm/llm_node.py b/api/core/workflow/nodes/llm/llm_node.py index eb8921b526..49f61bd597 100644 --- a/api/core/workflow/nodes/llm/llm_node.py +++ b/api/core/workflow/nodes/llm/llm_node.py @@ -109,7 +109,9 @@ class LLMNode(BaseNode): 'prompts': PromptMessageUtil.prompt_messages_to_prompt_for_saving( model_mode=model_config.mode, prompt_messages=prompt_messages - ) + ), + 'model_provider': model_config.provider, + 'model_name': model_config.model, } # handle invoke result From 7b2cf8215fb8418afae9faf30f2598697315ff59 Mon Sep 17 00:00:00 2001 From: Yuki Oshima <39944763+os1ma@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:44:59 +0900 Subject: [PATCH 16/22] chore: fix inverted index japanese translation (#7957) --- web/i18n/ja-JP/dataset.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/ja-JP/dataset.ts b/web/i18n/ja-JP/dataset.ts index d2eaf05276..5b5bd761e8 100644 --- a/web/i18n/ja-JP/dataset.ts +++ b/web/i18n/ja-JP/dataset.ts @@ -37,7 +37,7 @@ const translation = { recommend: 'おすすめ', }, invertedIndex: { - title: '逆インデックス', + title: '転置インデックス', description: '効率的な検索に使用される構造です。各用語が含まれるドキュメントまたはWebページを指すように、用語ごとに整理されています。', }, change: '変更', From 571415d1a4cd01078b4f46cb00f49c8daf923b3d Mon Sep 17 00:00:00 2001 From: Sumkor Date: Wed, 4 Sep 2024 12:59:10 +0800 Subject: [PATCH 17/22] fix: split text keep separator (#7930) --- api/core/rag/splitter/text_splitter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/core/rag/splitter/text_splitter.py b/api/core/rag/splitter/text_splitter.py index b3adcedc76..8ae5cbf7d1 100644 --- a/api/core/rag/splitter/text_splitter.py +++ b/api/core/rag/splitter/text_splitter.py @@ -30,15 +30,14 @@ def _split_text_with_regex( if keep_separator: # The parentheses in the pattern keep the delimiters in the result. _splits = re.split(f"({re.escape(separator)})", text) - splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)] - if len(_splits) % 2 == 0: + splits = [_splits[i - 1] + _splits[i] for i in range(1, len(_splits), 2)] + if len(_splits) % 2 != 0: splits += _splits[-1:] - splits = [_splits[0]] + splits else: splits = re.split(separator, text) else: splits = list(text) - return [s for s in splits if s != ""] + return [s for s in splits if (s != "" and s != '\n')] class TextSplitter(BaseDocumentTransformer, ABC): From 0bc19c3fbf1ba8faf238c96b3dcf9c071369180b Mon Sep 17 00:00:00 2001 From: Vico Chu <30412827+vicoooo26@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:03:06 +0800 Subject: [PATCH 18/22] Feat: update app published time after clicking publish button (#7801) --- .../components/app/app-publisher/index.tsx | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/web/app/components/app/app-publisher/index.tsx b/web/app/components/app/app-publisher/index.tsx index 0558e29956..2bcc74ec01 100644 --- a/web/app/components/app/app-publisher/index.tsx +++ b/web/app/components/app/app-publisher/index.tsx @@ -63,6 +63,7 @@ const AppPublisher = ({ const [published, setPublished] = useState(false) const [open, setOpen] = useState(false) const appDetail = useAppStore(state => state.appDetail) + const [publishedTime, setPublishedTime] = useState(publishedAt) const { app_base_url: appBaseURL = '', access_token: accessToken = '' } = appDetail?.site ?? {} const appMode = (appDetail?.mode !== 'completion' && appDetail?.mode !== 'workflow') ? 'chat' : appDetail.mode const appURL = `${appBaseURL}/${appMode}/${accessToken}` @@ -76,6 +77,7 @@ const AppPublisher = ({ try { await onPublish?.(modelAndParameter) setPublished(true) + setPublishedTime(Date.now()) } catch (e) { setPublished(false) @@ -131,13 +133,13 @@ const AppPublisher = ({
- {publishedAt ? t('workflow.common.latestPublished') : t('workflow.common.currentDraftUnpublished')} + {publishedTime ? t('workflow.common.latestPublished') : t('workflow.common.currentDraftUnpublished')}
- {publishedAt + {publishedTime ? (
- {t('workflow.common.publishedAt')} {formatTimeFromNow(publishedAt)} + {t('workflow.common.publishedAt')} {formatTimeFromNow(publishedTime)}
- }>{t('workflow.common.runApp')} + }>{t('workflow.common.runApp')} {appDetail?.mode === 'workflow' ? ( } > @@ -199,16 +201,16 @@ const AppPublisher = ({ setEmbeddingModalOpen(true) handleTrigger() }} - disabled={!publishedAt} + disabled={!publishedTime} icon={} > {t('workflow.common.embedIntoSite')} )} - }>{t('workflow.common.accessAPIReference')} + }>{t('workflow.common.accessAPIReference')} {appDetail?.mode === 'workflow' && ( Date: Wed, 4 Sep 2024 13:04:31 +0800 Subject: [PATCH 19/22] fix:empty voice occurs when xinference CosyVoice tts model (#7958) --- api/core/model_runtime/model_providers/xinference/tts/tts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/core/model_runtime/model_providers/xinference/tts/tts.py b/api/core/model_runtime/model_providers/xinference/tts/tts.py index bfa752df8c..8cc99fef7c 100644 --- a/api/core/model_runtime/model_providers/xinference/tts/tts.py +++ b/api/core/model_runtime/model_providers/xinference/tts/tts.py @@ -174,6 +174,11 @@ class XinferenceText2SpeechModel(TTSModel): return voices[language] elif 'all' in voices: return voices['all'] + else: + all_voices = [] + for lang, lang_voices in voices.items(): + all_voices.extend(lang_voices) + return all_voices return self.model_voices['__default']['all'] From c2a3c5a748caa7d8a508121c002cc9c31fcda7bc Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 4 Sep 2024 13:13:21 +0800 Subject: [PATCH 20/22] fix: get commit sha failed in translate action (#7959) --- .github/workflows/translate-i18n-base-on-english.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/translate-i18n-base-on-english.yml b/.github/workflows/translate-i18n-base-on-english.yml index 5b63306ef5..3f51b3b2c7 100644 --- a/.github/workflows/translate-i18n-base-on-english.yml +++ b/.github/workflows/translate-i18n-base-on-english.yml @@ -15,12 +15,14 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 2 # last 2 commits - name: Check for file changes in i18n/en-US id: check_files run: | - changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'i18n/en-US/*.ts') + recent_commit_sha=$(git rev-parse HEAD) + second_recent_commit_sha=$(git rev-parse HEAD~1) + changed_files=$(git diff --name-only $recent_commit_sha $second_recent_commit_sha -- 'i18n/en-US/*.ts') echo "Changed files: $changed_files" if [ -n "$changed_files" ]; then echo "FILES_CHANGED=true" >> $GITHUB_ENV From 83e84865be44bf57ce28e548ed6b531800a5c699 Mon Sep 17 00:00:00 2001 From: zhuhao <37029601+hwzhuhao@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:25:46 +0800 Subject: [PATCH 21/22] feat: add health check for pg and redis in docker-compose.middleware.yaml (#7961) (#7962) --- docker/docker-compose.middleware.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index 9a4c40448b..adaecc69ec 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -19,6 +19,11 @@ services: - ./volumes/db/data:/var/lib/postgresql/data ports: - "${EXPOSE_POSTGRES_PORT:-5432}:5432" + healthcheck: + test: [ "CMD", "pg_isready" ] + interval: 1s + timeout: 3s + retries: 30 # The redis cache. redis: @@ -31,6 +36,8 @@ services: command: redis-server --requirepass difyai123456 ports: - "${EXPOSE_REDIS_PORT:-6379}:6379" + healthcheck: + test: [ "CMD", "redis-cli", "ping" ] # The DifySandbox sandbox: From 14af87527f219d5bbbc04f47af5d16f56adec1fb Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Wed, 4 Sep 2024 14:41:47 +0800 Subject: [PATCH 22/22] Feat:remove estimation of embedding cost (#7950) Co-authored-by: jyong <718720800@qq.com> --- api/core/indexing_runner.py | 41 +------ api/core/rag/splitter/text_splitter.py | 23 ++-- api/services/dataset_service.py | 7 +- .../common/retrieval-method-config/index.tsx | 17 +++ .../create/embedding-process/index.tsx | 30 +---- .../datasets/create/step-two/index.module.css | 8 +- .../datasets/create/step-two/index.tsx | 111 +++++++++++------- .../datasets/create/steps-nav-bar/index.tsx | 2 +- .../documents/detail/embedding/index.tsx | 31 +---- .../detail/embedding/style.module.css | 2 +- web/i18n/en-US/dataset-creation.ts | 6 +- web/i18n/ja-JP/dataset-creation.ts | 2 +- web/i18n/zh-Hans/dataset-creation.ts | 2 +- web/models/datasets.ts | 2 + 14 files changed, 122 insertions(+), 162 deletions(-) diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 062666ac6a..7a1c5e760b 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -16,9 +16,7 @@ from configs import dify_config from core.errors.error import ProviderTokenNotInitError from core.llm_generator.llm_generator import LLMGenerator from core.model_manager import ModelInstance, ModelManager -from core.model_runtime.entities.model_entities import ModelType, PriceType -from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel -from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel +from core.model_runtime.entities.model_entities import ModelType from core.rag.datasource.keyword.keyword_factory import Keyword from core.rag.docstore.dataset_docstore import DatasetDocumentStore from core.rag.extractor.entity.extract_setting import ExtractSetting @@ -255,11 +253,8 @@ class IndexingRunner: tenant_id=tenant_id, model_type=ModelType.TEXT_EMBEDDING, ) - tokens = 0 preview_texts = [] total_segments = 0 - total_price = 0 - currency = 'USD' index_type = doc_form index_processor = IndexProcessorFactory(index_type).init_index_processor() all_text_docs = [] @@ -286,54 +281,22 @@ class IndexingRunner: for document in documents: if len(preview_texts) < 5: preview_texts.append(document.page_content) - if indexing_technique == 'high_quality' or embedding_model_instance: - tokens += embedding_model_instance.get_text_embedding_num_tokens( - texts=[self.filter_string(document.page_content)] - ) if doc_form and doc_form == 'qa_model': - model_instance = self.model_manager.get_default_model_instance( - tenant_id=tenant_id, - model_type=ModelType.LLM - ) - - model_type_instance = model_instance.model_type_instance - model_type_instance = cast(LargeLanguageModel, model_type_instance) if len(preview_texts) > 0: # qa model document response = LLMGenerator.generate_qa_document(current_user.current_tenant_id, preview_texts[0], doc_language) document_qa_list = self.format_split_text(response) - price_info = model_type_instance.get_price( - model=model_instance.model, - credentials=model_instance.credentials, - price_type=PriceType.INPUT, - tokens=total_segments * 2000, - ) + return { "total_segments": total_segments * 20, - "tokens": total_segments * 2000, - "total_price": '{:f}'.format(price_info.total_amount), - "currency": price_info.currency, "qa_preview": document_qa_list, "preview": preview_texts } - if embedding_model_instance: - embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_instance.model_type_instance) - embedding_price_info = embedding_model_type_instance.get_price( - model=embedding_model_instance.model, - credentials=embedding_model_instance.credentials, - price_type=PriceType.INPUT, - tokens=tokens - ) - total_price = '{:f}'.format(embedding_price_info.total_amount) - currency = embedding_price_info.currency return { "total_segments": total_segments, - "tokens": tokens, - "total_price": total_price, - "currency": currency, "preview": preview_texts } diff --git a/api/core/rag/splitter/text_splitter.py b/api/core/rag/splitter/text_splitter.py index 8ae5cbf7d1..943f9918a7 100644 --- a/api/core/rag/splitter/text_splitter.py +++ b/api/core/rag/splitter/text_splitter.py @@ -108,7 +108,7 @@ class TextSplitter(BaseDocumentTransformer, ABC): else: return text - def _merge_splits(self, splits: Iterable[str], separator: str) -> list[str]: + def _merge_splits(self, splits: Iterable[str], separator: str, lengths: list[int]) -> list[str]: # We now want to combine these smaller pieces into medium size # chunks to send to the LLM. separator_len = self._length_function(separator) @@ -116,8 +116,9 @@ class TextSplitter(BaseDocumentTransformer, ABC): docs = [] current_doc: list[str] = [] total = 0 + index = 0 for d in splits: - _len = self._length_function(d) + _len = lengths[index] if ( total + _len + (separator_len if len(current_doc) > 0 else 0) > self._chunk_size @@ -145,6 +146,7 @@ class TextSplitter(BaseDocumentTransformer, ABC): current_doc = current_doc[1:] current_doc.append(d) total += _len + (separator_len if len(current_doc) > 1 else 0) + index += 1 doc = self._join_docs(current_doc, separator) if doc is not None: docs.append(doc) @@ -493,11 +495,10 @@ class RecursiveCharacterTextSplitter(TextSplitter): self._separators = separators or ["\n\n", "\n", " ", ""] def _split_text(self, text: str, separators: list[str]) -> list[str]: - """Split incoming text and return chunks.""" final_chunks = [] - # Get appropriate separator to use separator = separators[-1] new_separators = [] + for i, _s in enumerate(separators): if _s == "": separator = _s @@ -508,25 +509,31 @@ class RecursiveCharacterTextSplitter(TextSplitter): break splits = _split_text_with_regex(text, separator, self._keep_separator) - # Now go merging things, recursively splitting longer texts. _good_splits = [] + _good_splits_lengths = [] # cache the lengths of the splits _separator = "" if self._keep_separator else separator + for s in splits: - if self._length_function(s) < self._chunk_size: + s_len = self._length_function(s) + if s_len < self._chunk_size: _good_splits.append(s) + _good_splits_lengths.append(s_len) else: if _good_splits: - merged_text = self._merge_splits(_good_splits, _separator) + merged_text = self._merge_splits(_good_splits, _separator, _good_splits_lengths) final_chunks.extend(merged_text) _good_splits = [] + _good_splits_lengths = [] if not new_separators: final_chunks.append(s) else: other_info = self._split_text(s, new_separators) final_chunks.extend(other_info) + if _good_splits: - merged_text = self._merge_splits(_good_splits, _separator) + merged_text = self._merge_splits(_good_splits, _separator, _good_splits_lengths) final_chunks.extend(merged_text) + return final_chunks def split_text(self, text: str) -> list[str]: diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index d10df3f600..ad552e1bab 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -1054,7 +1054,6 @@ class DocumentService: DocumentService.check_documents_upload_quota(count, features) - embedding_model = None dataset_collection_binding_id = None retrieval_model = None if document_data["indexing_technique"] == "high_quality": @@ -1082,10 +1081,10 @@ class DocumentService: tenant_id=tenant_id, name="", data_source_type=document_data["data_source"]["type"], - indexing_technique=document_data["indexing_technique"], + indexing_technique=document_data.get("indexing_technique", "high_quality"), created_by=account.id, - embedding_model=embedding_model.model if embedding_model else None, - embedding_model_provider=embedding_model.provider if embedding_model else None, + embedding_model=document_data.get("embedding_model"), + embedding_model_provider=document_data.get("embedding_model_provider"), collection_binding_id=dataset_collection_binding_id, retrieval_model=retrieval_model, ) diff --git a/web/app/components/datasets/common/retrieval-method-config/index.tsx b/web/app/components/datasets/common/retrieval-method-config/index.tsx index 1e407b62e1..20d93568ad 100644 --- a/web/app/components/datasets/common/retrieval-method-config/index.tsx +++ b/web/app/components/datasets/common/retrieval-method-config/index.tsx @@ -11,6 +11,11 @@ import { FileSearch02 } from '@/app/components/base/icons/src/vender/solid/files import { useProviderContext } from '@/context/provider-context' import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { + DEFAULT_WEIGHTED_SCORE, + RerankingModeEnum, + WeightedScoreEnum, +} from '@/models/datasets' type Props = { value: RetrievalConfig @@ -32,6 +37,18 @@ const RetrievalMethodConfig: FC = ({ reranking_provider_name: rerankDefaultModel?.provider.provider || '', reranking_model_name: rerankDefaultModel?.model || '', }, + reranking_mode: passValue.reranking_mode || (rerankDefaultModel ? RerankingModeEnum.RerankingModel : RerankingModeEnum.WeightedScore), + weights: passValue.weights || { + weight_type: WeightedScoreEnum.Customized, + vector_setting: { + vector_weight: DEFAULT_WEIGHTED_SCORE.other.semantic, + embedding_provider_name: '', + embedding_model_name: '', + }, + keyword_setting: { + keyword_weight: DEFAULT_WEIGHTED_SCORE.other.keyword, + }, + }, } } return passValue diff --git a/web/app/components/datasets/create/embedding-process/index.tsx b/web/app/components/datasets/create/embedding-process/index.tsx index 574dd083c7..7786582085 100644 --- a/web/app/components/datasets/create/embedding-process/index.tsx +++ b/web/app/components/datasets/create/embedding-process/index.tsx @@ -13,8 +13,7 @@ import cn from '@/utils/classnames' import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata' import Button from '@/app/components/base/button' import type { FullDocumentDetail, IndexingStatusResponse, ProcessRuleResponse } from '@/models/datasets' -import { formatNumber } from '@/utils/format' -import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchIndexingEstimateBatch, fetchProcessRule } from '@/service/datasets' +import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchProcessRule } from '@/service/datasets' import { DataSourceType } from '@/models/datasets' import NotionIcon from '@/app/components/base/notion-icon' import PriorityLabel from '@/app/components/billing/priority-label' @@ -142,14 +141,6 @@ const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], index }, apiParams => fetchProcessRule(omit(apiParams, 'action')), { revalidateOnFocus: false, }) - // get cost - const { data: indexingEstimateDetail } = useSWR({ - action: 'fetchIndexingEstimateBatch', - datasetId, - batchId, - }, apiParams => fetchIndexingEstimateBatch(omit(apiParams, 'action')), { - revalidateOnFocus: false, - }) const router = useRouter() const navToDocumentList = () => { @@ -190,28 +181,11 @@ const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], index return ( <> -
+
{isEmbedding && t('datasetDocuments.embedding.processing')} {isEmbeddingCompleted && t('datasetDocuments.embedding.completed')}
-
- {indexingType === 'high_quality' && ( -
-
- {t('datasetDocuments.embedding.highQuality')} · {t('datasetDocuments.embedding.estimate')} - {formatNumber(indexingEstimateDetail?.tokens || 0)}tokens - (${formatNumber(indexingEstimateDetail?.total_price || 0)}) -
- )} - {indexingType === 'economy' && ( -
-
- {t('datasetDocuments.embedding.economy')} · {t('datasetDocuments.embedding.estimate')} - 0tokens -
- )} -
{ enableBilling && plan.type !== Plan.team && ( diff --git a/web/app/components/datasets/create/step-two/index.module.css b/web/app/components/datasets/create/step-two/index.module.css index 24a62c8e3c..f89d6d67ea 100644 --- a/web/app/components/datasets/create/step-two/index.module.css +++ b/web/app/components/datasets/create/step-two/index.module.css @@ -30,7 +30,7 @@ } .indexItem { - min-height: 146px; + min-height: 126px; } .indexItem .disableMask { @@ -121,10 +121,6 @@ @apply pb-1; } -.radioItem.indexItem .typeHeader .tip { - @apply pb-3; -} - .radioItem .typeIcon { position: absolute; top: 18px; @@ -264,7 +260,7 @@ } .input { - @apply inline-flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal; + @apply inline-flex h-9 w-full py-1 px-2 pr-14 rounded-lg text-xs leading-normal; @apply bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-white placeholder:text-gray-400; } diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 10b378d8c5..b25ca71311 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -14,7 +14,7 @@ import PreviewItem, { PreviewType } from './preview-item' import LanguageSelect from './language-select' import s from './index.module.css' import cn from '@/utils/classnames' -import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' +import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' import { createDocument, createFirstDocument, @@ -41,8 +41,10 @@ import { IS_CE_EDITION } from '@/config' import { RETRIEVE_METHOD } from '@/types/app' import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' import Tooltip from '@/app/components/base/tooltip' -import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' +import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { LanguagesSupported } from '@/i18n/language' +import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' +import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' import { Globe01 } from '@/app/components/base/icons/src/vender/line/mapsAndTravel' @@ -109,7 +111,7 @@ const StepTwo = ({ const [previewScrolled, setPreviewScrolled] = useState(false) const [segmentationType, setSegmentationType] = useState(SegmentType.AUTO) const [segmentIdentifier, setSegmentIdentifier] = useState('\\n') - const [max, setMax] = useState(500) + const [max, setMax] = useState(5000) // default chunk length const [overlap, setOverlap] = useState(50) const [rules, setRules] = useState([]) const [defaultConfig, setDefaultConfig] = useState() @@ -131,7 +133,6 @@ const StepTwo = ({ const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean() const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState(null) const [automaticFileIndexingEstimate, setAutomaticFileIndexingEstimate] = useState(null) - const [estimateTokes, setEstimateTokes] = useState | null>(null) const fileIndexingEstimate = (() => { return segmentationType === SegmentType.AUTO ? automaticFileIndexingEstimate : customFileIndexingEstimate @@ -192,13 +193,10 @@ const StepTwo = ({ const fetchFileIndexingEstimate = async (docForm = DocForm.TEXT) => { // eslint-disable-next-line @typescript-eslint/no-use-before-define const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams(docForm)!) - if (segmentationType === SegmentType.CUSTOM) { + if (segmentationType === SegmentType.CUSTOM) setCustomFileIndexingEstimate(res) - } - else { + else setAutomaticFileIndexingEstimate(res) - indexType === IndexingType.QUALIFIED && setEstimateTokes({ tokens: res.tokens, total_price: res.total_price }) - } } const confirmChangeCustomConfig = () => { @@ -310,6 +308,19 @@ const StepTwo = ({ defaultModel: rerankDefaultModel, currentModel: isRerankDefaultModelVaild, } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank) + const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding) + const { data: defaultEmbeddingModel } = useDefaultModel(ModelTypeEnum.textEmbedding) + const [embeddingModel, setEmbeddingModel] = useState( + currentDataset?.embedding_model + ? { + provider: currentDataset.embedding_model_provider, + model: currentDataset.embedding_model, + } + : { + provider: defaultEmbeddingModel?.provider.provider || '', + model: defaultEmbeddingModel?.model || '', + }, + ) const getCreationParams = () => { let params if (segmentationType === SegmentType.CUSTOM && overlap > max) { @@ -324,6 +335,8 @@ const StepTwo = ({ process_rule: getProcessRule(), // eslint-disable-next-line @typescript-eslint/no-use-before-define retrieval_model: retrievalConfig, // Readonly. If want to changed, just go to settings page. + embedding_model: embeddingModel.model, // Readonly + embedding_model_provider: embeddingModel.provider, // Readonly } as CreateDocumentReq } else { // create @@ -360,6 +373,8 @@ const StepTwo = ({ doc_language: docLanguage, retrieval_model: postRetrievalConfig, + embedding_model: embeddingModel.model, + embedding_model_provider: embeddingModel.provider, } as CreateDocumentReq if (dataSourceType === DataSourceType.FILE) { params.data_source.info_list.file_info_list = { @@ -613,14 +628,17 @@ const StepTwo = ({
{t('datasetCreation.stepTwo.maxLength')}
- setMax(parseInt(e.target.value.replace(/^0+/, ''), 10))} - /> +
+ setMax(parseInt(e.target.value.replace(/^0+/, ''), 10))} + /> +
Tokens
+
@@ -635,14 +653,17 @@ const StepTwo = ({ } />
- setOverlap(parseInt(e.target.value.replace(/^0+/, ''), 10))} - /> +
+ setOverlap(parseInt(e.target.value.replace(/^0+/, ''), 10))} + /> +
Tokens
+
@@ -675,7 +696,7 @@ const StepTwo = ({ !isAPIKeySet && s.disabled, !hasSetIndexType && indexType === IndexingType.QUALIFIED && s.active, hasSetIndexType && s.disabled, - hasSetIndexType && '!w-full', + hasSetIndexType && '!w-full !min-h-[96px]', )} onClick={() => { if (isAPIKeySet) @@ -690,16 +711,6 @@ const StepTwo = ({ {!hasSetIndexType && {t('datasetCreation.stepTwo.recommend')}}
{t('datasetCreation.stepTwo.qualifiedTip')}
-
{t('datasetCreation.stepTwo.emstimateCost')}
- { - estimateTokes - ? ( -
{formatNumber(estimateTokes.tokens)} tokens(${formatNumber(estimateTokes.total_price)})
- ) - : ( -
{t('datasetCreation.stepTwo.calculating')}
- ) - }
{!isAPIKeySet && (
@@ -717,7 +728,7 @@ const StepTwo = ({ s.indexItem, !hasSetIndexType && indexType === IndexingType.ECONOMICAL && s.active, hasSetIndexType && s.disabled, - hasSetIndexType && '!w-full', + hasSetIndexType && '!w-full !min-h-[96px]', )} onClick={changeToEconomicalType} > @@ -726,13 +737,11 @@ const StepTwo = ({
{t('datasetCreation.stepTwo.economical')}
{t('datasetCreation.stepTwo.economicalTip')}
-
{t('datasetCreation.stepTwo.emstimateCost')}
-
0 tokens
)}
- {hasSetIndexType && ( + {hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
{t('datasetCreation.stepTwo.indexSettedTip')} {t('datasetCreation.stepTwo.datasetSettingLink')} @@ -767,12 +776,32 @@ const StepTwo = ({ )}
)} + {/* Embedding model */} + {indexType === IndexingType.QUALIFIED && ( +
+
{t('datasetSettings.form.embeddingModel')}
+ { + setEmbeddingModel(model) + }} + /> + {!!datasetId && ( +
+ {t('datasetCreation.stepTwo.indexSettedTip')} + {t('datasetCreation.stepTwo.datasetSettingLink')} +
+ )} +
+ )} {/* Retrieval Method Config */}
{!datasetId ? (
- {t('datasetSettings.form.retrievalSetting.title')} +
{t('datasetSettings.form.retrievalSetting.title')}
{t('datasetSettings.form.retrievalSetting.learnMore')} {t('datasetSettings.form.retrievalSetting.longDescription')} diff --git a/web/app/components/datasets/create/steps-nav-bar/index.tsx b/web/app/components/datasets/create/steps-nav-bar/index.tsx index 70724a308c..b676f3ace4 100644 --- a/web/app/components/datasets/create/steps-nav-bar/index.tsx +++ b/web/app/components/datasets/create/steps-nav-bar/index.tsx @@ -49,7 +49,7 @@ const StepsNavBar = ({ key={item} className={cn(s.stepItem, s[`step${item}`], step === item && s.active, step > item && s.done, isMobile && 'px-0')} > -
{item}
+
{step > item ? '' : item}
{isMobile ? '' : t(STEP_T_MAP[item])}
))} diff --git a/web/app/components/datasets/documents/detail/embedding/index.tsx b/web/app/components/datasets/documents/detail/embedding/index.tsx index 79b031549a..1bc6c91c2a 100644 --- a/web/app/components/datasets/documents/detail/embedding/index.tsx +++ b/web/app/components/datasets/documents/detail/embedding/index.tsx @@ -18,9 +18,7 @@ import { ToastContext } from '@/app/components/base/toast' import type { FullDocumentDetail, ProcessRuleResponse } from '@/models/datasets' import type { CommonResponse } from '@/models/common' import { asyncRunSafe, sleep } from '@/utils' -import { formatNumber } from '@/utils/format' -import { fetchIndexingStatus as doFetchIndexingStatus, fetchIndexingEstimate, fetchProcessRule, pauseDocIndexing, resumeDocIndexing } from '@/service/datasets' -import DatasetDetailContext from '@/context/dataset-detail' +import { fetchIndexingStatus as doFetchIndexingStatus, fetchProcessRule, pauseDocIndexing, resumeDocIndexing } from '@/service/datasets' import StopEmbeddingModal from '@/app/components/datasets/create/stop-embedding-modal' type Props = { @@ -108,16 +106,14 @@ const RuleDetail: FC<{ sourceData?: ProcessRuleResponse; docName?: string }> = (
} -const EmbeddingDetail: FC = ({ detail, stopPosition = 'top', datasetId: dstId, documentId: docId, indexingType, detailUpdate }) => { +const EmbeddingDetail: FC = ({ detail, stopPosition = 'top', datasetId: dstId, documentId: docId, detailUpdate }) => { const onTop = stopPosition === 'top' const { t } = useTranslation() const { notify } = useContext(ToastContext) const { datasetId = '', documentId = '' } = useContext(DocumentContext) - const { indexingTechnique } = useContext(DatasetDetailContext) const localDatasetId = dstId ?? datasetId const localDocumentId = docId ?? documentId - const localIndexingTechnique = indexingType ?? indexingTechnique const [indexingStatusDetail, setIndexingStatusDetail] = useState(null) const fetchIndexingStatus = async () => { @@ -160,14 +156,6 @@ const EmbeddingDetail: FC = ({ detail, stopPosition = 'top', datasetId: d } }, [startQueryStatus, stopQueryStatus]) - const { data: indexingEstimateDetail, error: indexingEstimateErr } = useSWR({ - action: 'fetchIndexingEstimate', - datasetId: localDatasetId, - documentId: localDocumentId, - }, apiParams => fetchIndexingEstimate(omit(apiParams, 'action')), { - revalidateOnFocus: false, - }) - const { data: ruleDetail, error: ruleError } = useSWR({ action: 'fetchProcessRule', params: { documentId: localDocumentId }, @@ -250,21 +238,6 @@ const EmbeddingDetail: FC = ({ detail, stopPosition = 'top', datasetId: d
{t('datasetDocuments.embedding.segments')} {indexingStatusDetail?.completed_segments}/{indexingStatusDetail?.total_segments} · {percent}%
- {localIndexingTechnique === 'high_quaility' && ( -
-
- {t('datasetDocuments.embedding.highQuality')} · {t('datasetDocuments.embedding.estimate')} - {formatNumber(indexingEstimateDetail?.tokens || 0)}tokens - (${formatNumber(indexingEstimateDetail?.total_price || 0)}) -
- )} - {localIndexingTechnique === 'economy' && ( -
-
- {t('datasetDocuments.embedding.economy')} · {t('datasetDocuments.embedding.estimate')} - 0tokens -
- )}
{!onTop && ( diff --git a/web/app/components/datasets/documents/detail/embedding/style.module.css b/web/app/components/datasets/documents/detail/embedding/style.module.css index 6dc1a5e80b..c24444ac12 100644 --- a/web/app/components/datasets/documents/detail/embedding/style.module.css +++ b/web/app/components/datasets/documents/detail/embedding/style.module.css @@ -31,7 +31,7 @@ @apply rounded-r-md; } .progressData { - @apply w-full flex justify-between items-center text-xs text-gray-700; + @apply w-full flex items-center text-xs text-gray-700; } .previewTip { @apply pb-1 pt-12 text-gray-900 text-sm font-medium; diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 1ead19f7e6..fa17dce81e 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -86,7 +86,7 @@ const translation = { autoDescription: 'Automatically set chunk and preprocessing rules. Unfamiliar users are recommended to select this.', custom: 'Custom', customDescription: 'Customize chunks rules, chunks length, and preprocessing rules, etc.', - separator: 'Segment identifier', + separator: 'Delimiter', separatorPlaceholder: 'For example, newline (\\\\n) or special separator (such as "***")', maxLength: 'Maximum chunk length', overlap: 'Chunk overlap', @@ -135,8 +135,8 @@ const translation = { previewSwitchTipStart: 'The current chunk preview is in text format, switching to a question-and-answer format preview will', previewSwitchTipEnd: ' consume additional tokens', characters: 'characters', - indexSettedTip: 'To change the index method, please go to the ', - retrivalSettedTip: 'To change the index method, please go to the ', + indexSettedTip: 'To change the index method & embedding model, please go to the ', + retrivalSettedTip: 'To change the retrieval setting, please go to the ', datasetSettingLink: 'Knowledge settings.', }, stepThree: { diff --git a/web/i18n/ja-JP/dataset-creation.ts b/web/i18n/ja-JP/dataset-creation.ts index 448e208222..5c90dcf925 100644 --- a/web/i18n/ja-JP/dataset-creation.ts +++ b/web/i18n/ja-JP/dataset-creation.ts @@ -136,7 +136,7 @@ const translation = { previewSwitchTipEnd: ' 追加のトークンが消費されます', characters: '文字', indexSettedTip: 'インデックス方法を変更するには、', - retrivalSettedTip: 'インデックス方法を変更するには、', + retrivalSettedTip: '検索方法を変更するには、', datasetSettingLink: 'ナレッジ設定', }, stepThree: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 257f409abd..53821e71a5 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -135,7 +135,7 @@ const translation = { previewSwitchTipStart: '当前分段预览是文本模式，切换到 Q&A 模式将会', previewSwitchTipEnd: '消耗额外的 token', characters: '字符', - indexSettedTip: '要更改索引方法，请转到', + indexSettedTip: '要更改索引方法和 embedding 模型，请转到', retrivalSettedTip: '要更改检索方法，请转到', datasetSettingLink: '知识库设置。', }, diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 0ae7831245..f35de58254 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -227,6 +227,8 @@ export type DocumentReq = { export type CreateDocumentReq = DocumentReq & { data_source: DataSource retrieval_model: RetrievalConfig + embedding_model: string + embedding_model_provider: string } export type IndexingEstimateParams = DocumentReq & Partial & {