Feat: Scrolling knowledge base list and set the number of entries per page to 30 #3695 (#3712)

### What problem does this PR solve? Feat: Scrolling knowledge base list and set the number of entries per page to 30 #3695 ### Type of change - [x] New Feature (non-breaking change which adds functionality)
2025-04-18 12:09:58 +08:00 · 2024-11-28 15:25:38 +08:00 · 2024-11-28 15:25:38 +08:00 · ec560cc99d
commit ec560cc99d
parent 7ae8828e61
12 changed files with 158 additions and 38 deletions
--- a/api/apps/kb_app.py
+++ b/api/apps/kb_app.py
@ -125,15 +125,16 @@ def detail():
@manager.route('/list', methods=['GET'])
@login_required
 def list_kbs():
+    keywords = request.args.get("keywords", "")
    page_number = int(request.args.get("page", 1))
    items_per_page = int(request.args.get("page_size", 150))
    orderby = request.args.get("orderby", "create_time")
    desc = request.args.get("desc", True)
    try:
        tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
-        kbs = KnowledgebaseService.get_by_tenant_ids(
-            [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc)
-        return get_json_result(data=kbs)
+        kbs, total = KnowledgebaseService.get_by_tenant_ids(
+            [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc, keywords)
+        return get_json_result(data={"kbs": kbs, "total": total})
    except Exception as e:
        return server_error_response(e)

--- a/api/db/services/knowledgebase_service.py
+++ b/api/db/services/knowledgebase_service.py
@ -16,6 +16,7 @@
 from api.db import StatusEnum, TenantPermission
 from api.db.db_models import Knowledgebase, DB, Tenant, User, UserTenant,Document
 from api.db.services.common_service import CommonService
+from peewee import fn


 class KnowledgebaseService(CommonService):
@ -34,7 +35,7 @@ class KnowledgebaseService(CommonService):
    @classmethod
    @DB.connection_context()
    def get_by_tenant_ids(cls, joined_tenant_ids, user_id,
-                          page_number, items_per_page, orderby, desc):
+                          page_number, items_per_page, orderby, desc, keywords):
        fields = [
            cls.model.id,
            cls.model.avatar,
@ -51,20 +52,31 @@ class KnowledgebaseService(CommonService):
            User.avatar.alias('tenant_avatar'),
            cls.model.update_time
        ]
-        kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
-            ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
-                                                            TenantPermission.TEAM.value)) | (
-                     cls.model.tenant_id == user_id))
-            & (cls.model.status == StatusEnum.VALID.value)
-        )
+        if keywords:
+            kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
+                ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
+                                                                TenantPermission.TEAM.value)) | (
+                    cls.model.tenant_id == user_id))
+                & (cls.model.status == StatusEnum.VALID.value),
+                (fn.LOWER(cls.model.name).contains(keywords.lower()))
+            )
+        else:
+            kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
+                ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
+                                                                TenantPermission.TEAM.value)) | (
+                    cls.model.tenant_id == user_id))
+                & (cls.model.status == StatusEnum.VALID.value)
+            )
        if desc:
            kbs = kbs.order_by(cls.model.getter_by(orderby).desc())
        else:
            kbs = kbs.order_by(cls.model.getter_by(orderby).asc())

+        count = kbs.count()
+
        kbs = kbs.paginate(page_number, items_per_page)

-        return list(kbs.dicts())
+        return list(kbs.dicts()), count

    @classmethod
    @DB.connection_context()
--- a/sdk/python/test/test_frontend_api/test_dataset.py
+++ b/sdk/python/test/test_frontend_api/test_dataset.py
@ -13,7 +13,7 @@ def test_dataset(get_auth):
    while True:
        res = list_dataset(get_auth, page_number)
        data = res.get("data")
-        for item in data:
+        for item in data.get("kbs"):
            dataset_id = item.get("id")
            dataset_list.append(dataset_id)
        if len(dataset_list) < page_number * 150:
@ -42,7 +42,7 @@ def test_dataset_1k_dataset(get_auth):
    while True:
        res = list_dataset(get_auth, page_number)
        data = res.get("data")
-        for item in data:
+        for item in data.get("kbs"):
            dataset_id = item.get("id")
            dataset_list.append(dataset_id)
        if len(dataset_list) < page_number * 150:
--- a/web/.umirc.ts
+++ b/web/.umirc.ts
@ -34,7 +34,7 @@ export default defineConfig({
  proxy: [
    {
      context: ['/api', '/v1'],
-      target: 'http://127.0.0.1:9456/',
+      target: 'http://127.0.0.1:9380/',
      changeOrigin: true,
      ws: true,
      logger: console,
--- a/web/package-lock.json
+++ b/web/package-lock.json
@ -57,6 +57,7 @@
        "react-force-graph": "^1.44.4",
        "react-hook-form": "^7.53.1",
        "react-i18next": "^14.0.0",
+        "react-infinite-scroll-component": "^6.1.0",
        "react-markdown": "^9.0.1",
        "react-pdf-highlighter": "^6.1.0",
        "react-string-replace": "^1.1.1",
@ -24705,6 +24706,25 @@
        }
      }
    },
+    "node_modules/react-infinite-scroll-component": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmmirror.com/react-infinite-scroll-component/-/react-infinite-scroll-component-6.1.0.tgz",
+      "integrity": "sha512-SQu5nCqy8DxQWpnUVLx7V7b7LcA37aM7tvoWjTLZp1dk6EJibM5/4EJKzOnl07/BsM1Y40sKLuqjCwwH/xV0TQ==",
+      "dependencies": {
+        "throttle-debounce": "^2.1.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.0.0"
+      }
+    },
+    "node_modules/react-infinite-scroll-component/node_modules/throttle-debounce": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmmirror.com/throttle-debounce/-/throttle-debounce-2.3.0.tgz",
+      "integrity": "sha512-H7oLPV0P7+jgvrk+6mwwwBDmxTaxnu9HMXmloNLXwnNO0ZxZ31Orah2n8lU1eMPvsaowP2CX+USCgyovXfdOFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/react-is": {
      "version": "18.2.0",
      "resolved": "https://registry.npmmirror.com/react-is/-/react-is-18.2.0.tgz",
--- a/web/package.json
+++ b/web/package.json
@ -68,6 +68,7 @@
    "react-force-graph": "^1.44.4",
    "react-hook-form": "^7.53.1",
    "react-i18next": "^14.0.0",
+    "react-infinite-scroll-component": "^6.1.0",
    "react-markdown": "^9.0.1",
    "react-pdf-highlighter": "^6.1.0",
    "react-string-replace": "^1.1.1",
--- a/web/src/hooks/knowledge-hooks.ts
+++ b/web/src/hooks/knowledge-hooks.ts
@ -3,14 +3,17 @@ import { IKnowledge, ITestingResult } from '@/interfaces/database/knowledge';
 import i18n from '@/locales/config';
 import kbService from '@/services/knowledge-service';
 import {
+  useInfiniteQuery,
  useIsMutating,
  useMutation,
  useMutationState,
  useQuery,
  useQueryClient,
 } from '@tanstack/react-query';
+import { useDebounce } from 'ahooks';
 import { message } from 'antd';
 import { useSearchParams } from 'umi';
+import { useHandleSearchChange } from './logic-hooks';
 import { useSetPaginationParams } from './route-hook';

 export const useKnowledgeBaseId = (): string => {
@ -50,7 +53,7 @@ export const useNextFetchKnowledgeList = (
    gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
    queryFn: async () => {
      const { data } = await kbService.getList();
-      const list = data?.data ?? [];
+      const list = data?.data?.kbs ?? [];
      return shouldFilterListWithoutDocument
        ? list.filter((x: IKnowledge) => x.chunk_num > 0)
        : list;
@ -60,6 +63,52 @@ export const useNextFetchKnowledgeList = (
  return { list: data, loading };
 };

+export const useInfiniteFetchKnowledgeList = () => {
+  const { searchString, handleInputChange } = useHandleSearchChange();
+  const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+
+  const PageSize = 30;
+  const {
+    data,
+    error,
+    fetchNextPage,
+    hasNextPage,
+    isFetching,
+    isFetchingNextPage,
+    status,
+  } = useInfiniteQuery({
+    queryKey: ['infiniteFetchKnowledgeList', debouncedSearchString],
+    queryFn: async ({ pageParam }) => {
+      const { data } = await kbService.getList({
+        page: pageParam,
+        page_size: PageSize,
+        keywords: debouncedSearchString,
+      });
+      const list = data?.data ?? [];
+      return list;
+    },
+    initialPageParam: 1,
+    getNextPageParam: (lastPage, pages, lastPageParam) => {
+      if (lastPageParam * PageSize <= lastPage.total) {
+        return lastPageParam + 1;
+      }
+      return undefined;
+    },
+  });
+  return {
+    data,
+    loading: isFetching,
+    error,
+    fetchNextPage,
+    hasNextPage,
+    isFetching,
+    isFetchingNextPage,
+    status,
+    handleInputChange,
+    searchString,
+  };
+};
+
 export const useCreateKnowledge = () => {
  const queryClient = useQueryClient();
  const {
@ -95,7 +144,9 @@ export const useDeleteKnowledge = () => {
      const { data } = await kbService.rmKb({ kb_id: id });
      if (data.code === 0) {
        message.success(i18n.t(`message.deleted`));
-        queryClient.invalidateQueries({ queryKey: ['fetchKnowledgeList'] });
+        queryClient.invalidateQueries({
+          queryKey: ['infiniteFetchKnowledgeList'],
+        });
      }
      return data?.data ?? [];
    },
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@ -75,6 +75,7 @@ export default {
      namePlaceholder: 'Please input name!',
      doc: 'Docs',
      searchKnowledgePlaceholder: 'Search',
+      noMoreData: 'It is all, nothing more',
    },
    knowledgeDetails: {
      dataset: 'Dataset',
--- a/web/src/locales/zh-traditional.ts
+++ b/web/src/locales/zh-traditional.ts
@ -75,6 +75,7 @@ export default {
      namePlaceholder: '請輸入名稱',
      doc: '文件',
      searchKnowledgePlaceholder: '搜索',
+      noMoreData: 'It is all, nothing more',
    },
    knowledgeDetails: {
      dataset: '數據集',
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@ -75,6 +75,7 @@ export default {
      namePlaceholder: '请输入名称',
      doc: '文档',
      searchKnowledgePlaceholder: '搜索',
+      noMoreData: '沒有更多的數據了',
    },
    knowledgeDetails: {
      dataset: '数据集',
--- a/web/src/pages/knowledge/index.less
+++ b/web/src/pages/knowledge/index.less
@ -2,6 +2,7 @@

 .knowledge {
  padding: 48px 0;
+  overflow: auto;
 }

 .topWrapper {
--- a/web/src/pages/knowledge/index.tsx
+++ b/web/src/pages/knowledge/index.tsx
@ -1,18 +1,26 @@
-import { useNextFetchKnowledgeList } from '@/hooks/knowledge-hooks';
+import { useInfiniteFetchKnowledgeList } from '@/hooks/knowledge-hooks';
 import { useFetchUserInfo } from '@/hooks/user-setting-hooks';
 import { PlusOutlined, SearchOutlined } from '@ant-design/icons';
-import { Button, Empty, Flex, Input, Space, Spin } from 'antd';
+import {
+  Button,
+  Divider,
+  Empty,
+  Flex,
+  Input,
+  Skeleton,
+  Space,
+  Spin,
+} from 'antd';
+import { useTranslation } from 'react-i18next';
+import InfiniteScroll from 'react-infinite-scroll-component';
+import { useSaveKnowledge } from './hooks';
 import KnowledgeCard from './knowledge-card';
 import KnowledgeCreatingModal from './knowledge-creating-modal';

-import { useTranslation } from 'react-i18next';
-import { useSaveKnowledge, useSearchKnowledge } from './hooks';
+import { useMemo } from 'react';
 import styles from './index.less';

 const KnowledgeList = () => {
-  const { searchString, handleInputChange } = useSearchKnowledge();
-  const { loading, list: data } = useNextFetchKnowledgeList();
-  const list = data.filter((x) => x.name.includes(searchString));
  const { data: userInfo } = useFetchUserInfo();
  const { t } = useTranslation('translation', { keyPrefix: 'knowledgeList' });
  const {
@ -22,9 +30,23 @@ const KnowledgeList = () => {
    onCreateOk,
    loading: creatingLoading,
  } = useSaveKnowledge();
+  const {
+    fetchNextPage,
+    data,
+    hasNextPage,
+    searchString,
+    handleInputChange,
+    loading,
+  } = useInfiniteFetchKnowledgeList();
+  console.log('🚀 ~ KnowledgeList ~ data:', data);
+  const nextList = data?.pages?.flatMap((x) => x.kbs) ?? [];
+
+  const total = useMemo(() => {
+    return data?.pages.at(-1).total ?? 0;
+  }, [data?.pages]);

  return (
-    <Flex className={styles.knowledge} vertical flex={1}>
+    <Flex className={styles.knowledge} vertical flex={1} id="scrollableDiv">
      <div className={styles.topWrapper}>
        <div>
          <span className={styles.title}>
@ -53,21 +75,30 @@ const KnowledgeList = () => {
        </Space>
      </div>
      <Spin spinning={loading}>
-        <Flex
-          gap={'large'}
-          wrap="wrap"
-          className={styles.knowledgeCardContainer}
+        <InfiniteScroll
+          dataLength={nextList?.length ?? 0}
+          next={fetchNextPage}
+          hasMore={hasNextPage}
+          loader={<Skeleton avatar paragraph={{ rows: 1 }} active />}
+          endMessage={total && <Divider plain>{t('noMoreData')} 🤐</Divider>}
+          scrollableTarget="scrollableDiv"
        >
-          {list.length > 0 ? (
-            list.map((item: any) => {
-              return (
-                <KnowledgeCard item={item} key={item.name}></KnowledgeCard>
-              );
-            })
-          ) : (
-            <Empty className={styles.knowledgeEmpty}></Empty>
-          )}
-        </Flex>
+          <Flex
+            gap={'large'}
+            wrap="wrap"
+            className={styles.knowledgeCardContainer}
+          >
+            {nextList?.length > 0 ? (
+              nextList.map((item: any) => {
+                return (
+                  <KnowledgeCard item={item} key={item.name}></KnowledgeCard>
+                );
+              })
+            ) : (
+              <Empty className={styles.knowledgeEmpty}></Empty>
+            )}
+          </Flex>
+        </InfiniteScroll>
      </Spin>
      <KnowledgeCreatingModal
        loading={creatingLoading}