diff --git a/web/src/components/chunk-method-dialog/hooks.ts b/web/src/components/chunk-method-dialog/hooks.ts new file mode 100644 index 000000000..3e00161d6 --- /dev/null +++ b/web/src/components/chunk-method-dialog/hooks.ts @@ -0,0 +1,128 @@ +import { useSelectParserList } from '@/hooks/user-setting-hooks'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +const ParserListMap = new Map([ + [ + ['pdf'], + [ + 'naive', + 'resume', + 'manual', + 'paper', + 'book', + 'laws', + 'presentation', + 'one', + 'qa', + 'knowledge_graph', + ], + ], + [ + ['doc', 'docx'], + [ + 'naive', + 'resume', + 'book', + 'laws', + 'one', + 'qa', + 'manual', + 'knowledge_graph', + ], + ], + [ + ['xlsx', 'xls'], + ['naive', 'qa', 'table', 'one', 'knowledge_graph'], + ], + [['ppt', 'pptx'], ['presentation']], + [ + ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tif', 'tiff', 'webp', 'svg', 'ico'], + ['picture'], + ], + [ + ['txt'], + [ + 'naive', + 'resume', + 'book', + 'laws', + 'one', + 'qa', + 'table', + 'knowledge_graph', + ], + ], + [ + ['csv'], + [ + 'naive', + 'resume', + 'book', + 'laws', + 'one', + 'qa', + 'table', + 'knowledge_graph', + ], + ], + [['md'], ['naive', 'qa', 'knowledge_graph']], + [['json'], ['naive', 'knowledge_graph']], + [['eml'], ['email']], +]); + +const getParserList = ( + values: string[], + parserList: Array<{ + value: string; + label: string; + }>, +) => { + return parserList.filter((x) => values?.some((y) => y === x.value)); +}; + +export const useFetchParserListOnMount = ( + documentId: string, + parserId: string, + documentExtension: string, + // form: FormInstance, +) => { + const [selectedTag, setSelectedTag] = useState(''); + const parserList = useSelectParserList(); + // const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form); // TODO + + const nextParserList = useMemo(() => { + const key = [...ParserListMap.keys()].find((x) => + x.some((y) => y === documentExtension), + ); + if (key) { + const values = ParserListMap.get(key); + return getParserList(values ?? [], parserList); + } + + return getParserList( + ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table'], + parserList, + ); + }, [parserList, documentExtension]); + + useEffect(() => { + setSelectedTag(parserId); + }, [parserId, documentId]); + + const handleChange = (tag: string) => { + // handleChunkMethodSelectChange(tag); + setSelectedTag(tag); + }; + + return { parserList: nextParserList, handleChange, selectedTag }; +}; + +const hideAutoKeywords = ['qa', 'table', 'resume', 'knowledge_graph', 'tag']; + +export const useShowAutoKeywords = () => { + const showAutoKeywords = useCallback((selectedTag: string) => { + return hideAutoKeywords.every((x) => selectedTag !== x); + }, []); + + return showAutoKeywords; +}; diff --git a/web/src/components/chunk-method-dialog/index.tsx b/web/src/components/chunk-method-dialog/index.tsx new file mode 100644 index 000000000..add64edba --- /dev/null +++ b/web/src/components/chunk-method-dialog/index.tsx @@ -0,0 +1,127 @@ +import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { useTranslate } from '@/hooks/common-hooks'; +import { IModalProps } from '@/interfaces/common'; +import { IParserConfig } from '@/interfaces/database/document'; +import { IChangeParserConfigRequestBody } from '@/interfaces/request/document'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { useForm } from 'react-hook-form'; +import { z } from 'zod'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '../ui/select'; +import { useFetchParserListOnMount } from './hooks'; + +interface IProps + extends IModalProps<{ + parserId: string; + parserConfig: IChangeParserConfigRequestBody; + }> { + loading: boolean; + parserId: string; + parserConfig: IParserConfig; + documentExtension: string; + documentId: string; +} + +export function ChunkMethodDialog({ + hideModal, + onOk, + parserId, + documentId, + documentExtension, +}: IProps) { + const { t } = useTranslate('knowledgeDetails'); + + const { parserList } = useFetchParserListOnMount( + documentId, + parserId, + documentExtension, + // form, + ); + + const FormSchema = z.object({ + name: z + .string() + .min(1, { + message: 'namePlaceholder', + }) + .trim(), + }); + const form = useForm>({ + resolver: zodResolver(FormSchema), + defaultValues: { name: '' }, + }); + + async function onSubmit(data: z.infer) { + const ret = await onOk?.(); + if (ret) { + hideModal?.(); + } + } + + return ( + + + + {t('chunkMethod')} + +
+ + ( + + {t('name')} + + + + + + )} + /> + + + + + +
+
+ ); +} diff --git a/web/src/hooks/document-hooks.ts b/web/src/hooks/document-hooks.ts index e20a03d18..ef7ed1869 100644 --- a/web/src/hooks/document-hooks.ts +++ b/web/src/hooks/document-hooks.ts @@ -16,6 +16,7 @@ import { UploadFile, message } from 'antd'; import { get } from 'lodash'; import { useCallback, useMemo, useState } from 'react'; import { IHighlight } from 'react-pdf-highlighter'; +import { useParams } from 'umi'; import { useGetPaginationWithRouter, useHandleSearchChange, @@ -61,6 +62,7 @@ export const useFetchNextDocumentList = () => { const { knowledgeId } = useGetKnowledgeSearchParams(); const { searchString, handleInputChange } = useHandleSearchChange(); const { pagination, setPagination } = useGetPaginationWithRouter(); + const { id } = useParams(); const { data, isFetching: loading } = useQuery<{ docs: IDocumentInfo[]; @@ -69,9 +71,10 @@ export const useFetchNextDocumentList = () => { queryKey: ['fetchDocumentList', searchString, pagination], initialData: { docs: [], total: 0 }, refetchInterval: 15000, + enabled: !!knowledgeId || !!id, queryFn: async () => { const ret = await kbService.get_document_list({ - kb_id: knowledgeId, + kb_id: knowledgeId || id, keywords: searchString, page_size: pagination.pageSize, page: pagination.current, diff --git a/web/src/hooks/logic-hooks/navigate-hooks.ts b/web/src/hooks/logic-hooks/navigate-hooks.ts index 2deae037d..1bedbb774 100644 --- a/web/src/hooks/logic-hooks/navigate-hooks.ts +++ b/web/src/hooks/logic-hooks/navigate-hooks.ts @@ -9,9 +9,12 @@ export const useNavigatePage = () => { navigate(Routes.Datasets); }, [navigate]); - const navigateToDataset = useCallback(() => { - navigate(Routes.Dataset); - }, [navigate]); + const navigateToDataset = useCallback( + (id: string) => () => { + navigate(`${Routes.Dataset}/${id}`); + }, + [navigate], + ); const navigateToHome = useCallback(() => { navigate(Routes.Home); diff --git a/web/src/pages/dataset/dataset/dataset-table.tsx b/web/src/pages/dataset/dataset/dataset-table.tsx index 3856a2872..4cea3e098 100644 --- a/web/src/pages/dataset/dataset/dataset-table.tsx +++ b/web/src/pages/dataset/dataset/dataset-table.tsx @@ -1,7 +1,6 @@ 'use client'; import { - ColumnDef, ColumnFiltersState, SortingState, VisibilityState, @@ -12,20 +11,10 @@ import { getSortedRowModel, useReactTable, } from '@tanstack/react-table'; -import { ArrowUpDown, MoreHorizontal, Pencil } from 'lucide-react'; import * as React from 'react'; +import { ChunkMethodDialog } from '@/components/chunk-method-dialog'; import { Button } from '@/components/ui/button'; -import { Checkbox } from '@/components/ui/checkbox'; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuLabel, - DropdownMenuSeparator, - DropdownMenuTrigger, -} from '@/components/ui/dropdown-menu'; -import { Switch } from '@/components/ui/switch'; import { Table, TableBody, @@ -34,43 +23,22 @@ import { TableHeader, TableRow, } from '@/components/ui/table'; -import { RunningStatus } from '@/constants/knowledge'; +import { useFetchNextDocumentList } from '@/hooks/document-hooks'; +import { useSetSelectedRecord } from '@/hooks/logic-hooks'; import { IDocumentInfo } from '@/interfaces/database/document'; -import { useTranslation } from 'react-i18next'; - -const data: IDocumentInfo[] = [ - { - chunk_num: 1, - create_date: 'Thu, 28 Nov 2024 17:10:22 GMT', - create_time: 1732785022792, - created_by: 'b0975cb4bc3111ee9b830aef05f5e94f', - id: '990cb30ead6811efb9b9fa163e197198', - kb_id: '25a8cfbe9cd411efbc12fa163e197198', - location: 'mian.jpg', - name: 'mian.jpg', - parser_config: { - pages: [[1, 1000000]], - }, - parser_id: 'picture', - process_begin_at: 'Thu, 28 Nov 2024 17:10:25 GMT', - process_duation: 8.46185, - progress: 1, - progress_msg: - '\nTask has been received.\nPage(1~100000001): Finish OCR: (用小麦粉\n金\nONGXI ...)\nPage(1~100000001): OCR results is too long to use CV LLM.\nPage(1~100000001): Finished slicing files (1 chunks in 0.34s). Start to embedding the content.\nPage(1~100000001): Finished embedding (in 0.35s)! Start to build index!\nPage(1~100000001): Indexing elapsed in 0.02s.\nPage(1~100000001): Done!', - run: RunningStatus.RUNNING, - size: 19692, - source_type: 'local', - status: '1', - thumbnail: - '/v1/document/image/25a8cfbe9cd411efbc12fa163e197198-thumbnail_990cb30ead6811efb9b9fa163e197198.png', - token_num: 115, - type: 'visual', - update_date: 'Thu, 28 Nov 2024 17:10:33 GMT', - update_time: 1732785033462, - }, -]; +import { getExtension } from '@/utils/document-util'; +import { useMemo } from 'react'; +import { useChangeDocumentParser } from './hooks'; +import { useDatasetTableColumns } from './use-dataset-table-columns'; export function DatasetTable() { + const { + // searchString, + documents, + pagination, + // handleInputChange, + setPagination, + } = useFetchNextDocumentList(); const [sorting, setSorting] = React.useState([]); const [columnFilters, setColumnFilters] = React.useState( [], @@ -78,122 +46,31 @@ export function DatasetTable() { const [columnVisibility, setColumnVisibility] = React.useState({}); const [rowSelection, setRowSelection] = React.useState({}); - const { t } = useTranslation('translation', { - keyPrefix: 'knowledgeDetails', + + const { currentRecord, setRecord } = useSetSelectedRecord(); + + const { + changeParserLoading, + onChangeParserOk, + changeParserVisible, + hideChangeParserModal, + showChangeParserModal, + } = useChangeDocumentParser(currentRecord.id); + + const columns = useDatasetTableColumns({ + showChangeParserModal, + setCurrentRecord: setRecord, }); - const columns: ColumnDef[] = [ - { - id: 'select', - header: ({ table }) => ( - table.toggleAllPageRowsSelected(!!value)} - aria-label="Select all" - /> - ), - cell: ({ row }) => ( - row.toggleSelected(!!value)} - aria-label="Select row" - /> - ), - enableSorting: false, - enableHiding: false, - }, - { - accessorKey: 'name', - header: ({ column }) => { - return ( - - ); - }, - cell: ({ row }) => ( -
{row.getValue('name')}
- ), - }, - { - accessorKey: 'create_time', - header: ({ column }) => { - return ( - - ); - }, - cell: ({ row }) => ( -
{row.getValue('create_time')}
- ), - }, - { - accessorKey: 'parser_id', - header: t('chunkMethod'), - cell: ({ row }) => ( -
{row.getValue('parser_id')}
- ), - }, - { - accessorKey: 'run', - header: t('parsingStatus'), - cell: ({ row }) => ( - - ), - }, - { - id: 'actions', - header: t('action'), - enableHiding: false, - cell: ({ row }) => { - const payment = row.original; - - return ( -
- - - - - - - - Actions - navigator.clipboard.writeText(payment.id)} - > - Copy payment ID - - - View customer - View payment details - - -
- ); - }, - }, - ]; + const currentPagination = useMemo(() => { + return { + pageIndex: (pagination.current || 1) - 1, + pageSize: pagination.pageSize || 10, + }; + }, [pagination]); const table = useReactTable({ - data, + data: documents, columns, onSortingChange: setSorting, onColumnFiltersChange: setColumnFilters, @@ -203,12 +80,29 @@ export function DatasetTable() { getFilteredRowModel: getFilteredRowModel(), onColumnVisibilityChange: setColumnVisibility, onRowSelectionChange: setRowSelection, + onPaginationChange: (updaterOrValue: any) => { + if (typeof updaterOrValue === 'function') { + const nextPagination = updaterOrValue(currentPagination); + setPagination({ + page: nextPagination.pageIndex + 1, + pageSize: nextPagination.pageSize, + }); + } else { + setPagination({ + page: updaterOrValue.pageIndex, + pageSize: updaterOrValue.pageSize, + }); + } + }, + manualPagination: true, //we're doing manual "server-side" pagination state: { sorting, columnFilters, columnVisibility, rowSelection, + pagination: currentPagination, }, + rowCount: pagination.total ?? 0, }); return ( @@ -241,7 +135,10 @@ export function DatasetTable() { data-state={row.getIsSelected() && 'selected'} > {row.getVisibleCells().map((cell) => ( - + {flexRender( cell.column.columnDef.cell, cell.getContext(), @@ -266,7 +163,7 @@ export function DatasetTable() {
{table.getFilteredSelectedRowModel().rows.length} of{' '} - {table.getFilteredRowModel().rows.length} row(s) selected. + {pagination?.total} row(s) selected.
+ {changeParserVisible && ( + + )} ); } diff --git a/web/src/pages/dataset/dataset/hooks.ts b/web/src/pages/dataset/dataset/hooks.ts index c847cac7c..f94721edd 100644 --- a/web/src/pages/dataset/dataset/hooks.ts +++ b/web/src/pages/dataset/dataset/hooks.ts @@ -93,7 +93,13 @@ export const useChangeDocumentParser = (documentId: string) => { } = useSetModalState(); const onChangeParserOk = useCallback( - async (parserId: string, parserConfig: IChangeParserConfigRequestBody) => { + async ({ + parserId, + parserConfig, + }: { + parserId: string; + parserConfig: IChangeParserConfigRequestBody; + }) => { const ret = await setDocumentParser({ parserId, documentId, diff --git a/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx b/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx new file mode 100644 index 000000000..b46316075 --- /dev/null +++ b/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx @@ -0,0 +1,194 @@ +import SvgIcon from '@/components/svg-icon'; +import { Button } from '@/components/ui/button'; +import { Checkbox } from '@/components/ui/checkbox'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; +import { Switch } from '@/components/ui/switch'; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from '@/components/ui/tooltip'; +import { IDocumentInfo } from '@/interfaces/database/document'; +import { cn } from '@/lib/utils'; +import { getExtension } from '@/utils/document-util'; +import { ColumnDef } from '@tanstack/table-core'; +import { ArrowUpDown, MoreHorizontal, Pencil, Wrench } from 'lucide-react'; +import { useCallback } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useChangeDocumentParser } from './hooks'; + +type UseDatasetTableColumnsType = Pick< + ReturnType, + 'showChangeParserModal' +> & { setCurrentRecord: (record: IDocumentInfo) => void }; + +export function useDatasetTableColumns({ + showChangeParserModal, + setCurrentRecord, +}: UseDatasetTableColumnsType) { + const { t } = useTranslation('translation', { + keyPrefix: 'knowledgeDetails', + }); + + // const onShowRenameModal = (record: IDocumentInfo) => { + // setCurrentRecord(record); + // showRenameModal(); + // }; + const onShowChangeParserModal = useCallback( + (record: IDocumentInfo) => () => { + setCurrentRecord(record); + showChangeParserModal(); + }, + [setCurrentRecord, showChangeParserModal], + ); + + // const onShowSetMetaModal = useCallback(() => { + // setRecord(); + // showSetMetaModal(); + // }, [setRecord, showSetMetaModal]); + + const columns: ColumnDef[] = [ + { + id: 'select', + header: ({ table }) => ( + table.toggleAllPageRowsSelected(!!value)} + aria-label="Select all" + /> + ), + cell: ({ row }) => ( + row.toggleSelected(!!value)} + aria-label="Select row" + /> + ), + enableSorting: false, + enableHiding: false, + }, + { + accessorKey: 'name', + header: ({ column }) => { + return ( + + ); + }, + meta: { cellClassName: 'max-w-[20vw]' }, + cell: ({ row }) => { + const name: string = row.getValue('name'); + // return
{row.getValue('name')}
; + + return ( + + +
+ + {name} +
+
+ +

{name}

+
+
+ ); + }, + }, + { + accessorKey: 'create_time', + header: ({ column }) => { + return ( + + ); + }, + cell: ({ row }) => ( +
{row.getValue('create_time')}
+ ), + }, + { + accessorKey: 'parser_id', + header: t('chunkMethod'), + cell: ({ row }) => ( +
{row.getValue('parser_id')}
+ ), + }, + { + accessorKey: 'run', + header: t('parsingStatus'), + cell: ({ row }) => ( + + ), + }, + { + id: 'actions', + header: t('action'), + enableHiding: false, + cell: ({ row }) => { + const record = row.original; + + return ( +
+ + + + + + + + + Actions + navigator.clipboard.writeText(record.id)} + > + Copy payment ID + + + View customer + View payment details + + +
+ ); + }, + }, + ]; + + return columns; +} diff --git a/web/src/pages/datasets/index.tsx b/web/src/pages/datasets/index.tsx index 54503a1cf..792289d64 100644 --- a/web/src/pages/datasets/index.tsx +++ b/web/src/pages/datasets/index.tsx @@ -1,87 +1,17 @@ import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog'; import ListFilterBar from '@/components/list-filter-bar'; +import { Avatar, AvatarFallback, AvatarImage } from '@/components/ui/avatar'; import { Button } from '@/components/ui/button'; import { Card, CardContent } from '@/components/ui/card'; +import { useInfiniteFetchKnowledgeList } from '@/hooks/knowledge-hooks'; import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks'; +import { IKnowledge } from '@/interfaces/database/knowledge'; +import { formatDate } from '@/utils/date'; import { ChevronRight, Plus, Trash2 } from 'lucide-react'; +import { useMemo } from 'react'; import { DatasetCreatingDialog } from './dataset-creating-dialog'; import { useSaveKnowledge } from './hooks'; -const datasets = [ - { - id: 1, - title: 'Legal knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, - { - id: 2, - title: 'HR knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, - { - id: 3, - title: 'IT knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, - { - id: 4, - title: 'Legal knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, - { - id: 5, - title: 'Legal knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, - { - id: 6, - title: 'Legal knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, - { - id: 7, - title: 'Legal knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, - { - id: 8, - title: 'Legal knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, - { - id: 9, - title: 'Legal knowledge base', - files: '1,242 files', - size: '152 MB', - created: '12.02.2024', - image: 'https://github.com/shadcn.png', - }, -]; - export default function Datasets() { const { visible, @@ -92,6 +22,25 @@ export default function Datasets() { } = useSaveKnowledge(); const { navigateToDataset } = useNavigatePage(); + const { + fetchNextPage, + data, + hasNextPage, + searchString, + handleInputChange, + loading, + } = useInfiniteFetchKnowledgeList(); + + const nextList: IKnowledge[] = useMemo(() => { + const list = + data?.pages?.flatMap((x) => (Array.isArray(x.kbs) ? x.kbs : [])) ?? []; + return list; + }, [data?.pages]); + + const total = useMemo(() => { + return data?.pages.at(-1).total ?? 0; + }, [data?.pages]); + return (
@@ -99,17 +48,17 @@ export default function Datasets() { Create dataset
- {datasets.map((dataset) => ( + {nextList.map((dataset) => (
-
+ + + CN +
-

- {dataset.title} -

+

{dataset.name}

+

{dataset.doc_num} files

- {dataset.files} | {dataset.size} -

-

- Created {dataset.created} + Created {formatDate(dataset.update_time)}

diff --git a/web/src/pages/home/datasets.tsx b/web/src/pages/home/datasets.tsx index fa862484e..d1c4cf7c5 100644 --- a/web/src/pages/home/datasets.tsx +++ b/web/src/pages/home/datasets.tsx @@ -58,7 +58,7 @@ export function Datasets() { diff --git a/web/src/routes.ts b/web/src/routes.ts index aa3eb9a3a..8cdaf3860 100644 --- a/web/src/routes.ts +++ b/web/src/routes.ts @@ -222,7 +222,7 @@ const routes = [ component: `@/pages${Routes.DatasetBase}`, routes: [ { - path: Routes.Dataset, + path: `${Routes.Dataset}/:id`, component: `@/pages${Routes.Dataset}`, }, {