'use client' import type { FC } from 'react' import React, { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import s from './index.module.css' import NoData from './no-data' import Firecrawl from './firecrawl' import Watercrawl from './watercrawl' import JinaReader from './jina-reader' import cn from '@/utils/classnames' import { useModalContext } from '@/context/modal-context' import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import { fetchDataSources } from '@/service/datasets' import { type DataSourceItem, DataSourceProvider } from '@/models/common' import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config' type Props = { onPreview: (payload: CrawlResultItem) => void checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onCrawlProviderChange: (provider: DataSourceProvider) => void onJobIdChange: (jobId: string) => void crawlOptions: CrawlOptions onCrawlOptionsChange: (payload: CrawlOptions) => void } const Website: FC = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, onCrawlProviderChange, onJobIdChange, crawlOptions, onCrawlOptionsChange, }) => { const { t } = useTranslation() const { setShowAccountSettingModal } = useModalContext() const [isLoaded, setIsLoaded] = useState(false) const [selectedProvider, setSelectedProvider] = useState(DataSourceProvider.jinaReader) const [sources, setSources] = useState([]) useEffect(() => { onCrawlProviderChange(selectedProvider) }, [selectedProvider, onCrawlProviderChange]) const checkSetApiKey = useCallback(async () => { const res = await fetchDataSources() as any setSources(res.sources) // If users have configured one of the providers, select it. const availableProviders = res.sources.filter((item: DataSourceItem) => [ DataSourceProvider.jinaReader, DataSourceProvider.fireCrawl, DataSourceProvider.waterCrawl, ].includes(item.provider), ) if (availableProviders.length > 0) setSelectedProvider(availableProviders[0].provider) }, []) useEffect(() => { checkSetApiKey().then(() => { setIsLoaded(true) }) // eslint-disable-next-line react-hooks/exhaustive-deps }, []) const handleOnConfig = useCallback(() => { setShowAccountSettingModal({ payload: 'data-source', onCancelCallback: checkSetApiKey, }) }, [checkSetApiKey, setShowAccountSettingModal]) if (!isLoaded) return null const source = sources.find(source => source.provider === selectedProvider) return (
{t('datasetCreation.stepOne.website.chooseProvider')}
{ENABLE_WEBSITE_JINAREADER && } {ENABLE_WEBSITE_FIRECRAWL && } {ENABLE_WEBSITE_WATERCRAWL && }
{source && selectedProvider === DataSourceProvider.fireCrawl && ( )} {source && selectedProvider === DataSourceProvider.waterCrawl && ( )} {source && selectedProvider === DataSourceProvider.jinaReader && ( )} {!source && ( )}
) } export default React.memo(Website)