feat: add Wikipedia operator #918 (#1516)

### What problem does this PR solve? Add Wikipedia operator #918 ### Type of change - [x] New Feature (non-breaking change which adds functionality)
2025-06-01 09:12:49 +08:00 · 2024-07-15 17:48:01 +08:00 · 2024-07-15 17:48:01 +08:00 · 1daa0b4d46
commit 1daa0b4d46
parent 60d406acaa
8 changed files with 342 additions and 0 deletions
--- a/web/src/assets/svg/wikipedia.svg
+++ b/web/src/assets/svg/wikipedia.svg
@ -0,0 +1,6 @@
+<svg t="1721036025186" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4241"
+    width="200" height="200">
+    <path
+        d="M515.84 559.744c-39.936 82.432-94.592 194.048-121.728 244.394667-26.282667 45.824-48.085333 39.722667-65.365333 1.237333-59.989333-141.696-183.168-390.144-241.109334-529.450667-10.709333-25.642667-18.816-42.112-26.410666-48.597333-7.722667-6.4-23.637333-10.24-47.872-11.562667C4.394667 214.741333 0 212.565333 0 208.981333v-19.413333l2.218667-1.92c39.424-0.213333 230.442667 0 230.442666 0l2.176 1.92v18.517333c0 5.077333-3.2 7.509333-9.6 7.509334l-24.064 1.322666c-20.693333 1.237333-31.018667 6.997333-31.018666 18.602667 0 5.76 2.261333 14.08 7.082666 25.642667 46.165333 112.896 205.568 448.896 205.568 448.896l5.802667 1.962666 102.869333-205.226666-20.565333-45.525334-70.741333-139.264s-13.568-27.904-18.261334-37.205333c-31.061333-61.568-30.378667-64.768-61.738666-68.992-8.832-0.981333-13.354667-2.133333-13.354667-6.357333v-19.968l2.56-1.92h183.125333l4.821334 1.578666v19.242667c0 4.48-3.242667 6.4-9.685334 6.4l-13.141333 2.005333c-33.792 2.602667-28.202667 16.256-5.802667 60.672l67.498667 138.752 75.008-149.504c12.501333-27.306667 9.941333-34.176 4.736-40.405333-2.986667-3.584-13.013333-9.386667-34.645333-10.24l-8.576-0.896a9.728 9.728 0 0 1-6.186667-2.176 6.4 6.4 0 0 1-2.858667-5.504v-18.218667l2.602667-1.92c53.205333-0.341333 172.501333 0 172.501333 0l2.517334 1.92v18.602667c0 5.162667-2.517333 7.594667-8.234667 7.594667-27.562667 1.28-33.365333 4.053333-43.648 18.730666-5.12 7.936-16 25.130667-27.562667 44.330667l-98.176 182.314667-2.773333 5.76 119.125333 243.712 7.253334 2.048 187.562666-445.354667c6.570667-18.005333 5.504-30.805333-2.730666-38.186667-8.405333-7.338667-14.762667-11.648-36.565334-12.586666l-17.92-0.682667a10.88 10.88 0 0 1-6.485333-1.92c-1.834667-1.237333-3.072-3.2-3.072-5.077333v-18.602667l2.517333-1.92h211.669334l1.749333 1.92v18.645333c0 5.077333-3.157333 7.68-8.917333 7.68-27.648 1.28-48.085333 7.68-61.568 17.962667-13.397333 10.88-23.765333 26.282667-31.402667 45.525333 0 0-172.501333 395.008-231.509333 526.464-22.4 42.965333-44.928 39.125333-64.128-1.322666-24.362667-49.962667-75.648-161.536-112.896-243.626667l2.261333-1.536z"
+        fill="" p-id="4242"></path>
+</svg>
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@ -579,6 +579,7 @@ The above is the content you need to summarize.`,
      messageDescription:
        'This component is used to send user static information. You can prepare several messages which will be chosen randomly.',
      keywordDescription: `This component is used to extract keywords from user's question. Top N specifies the number of keywords you need to extract.`,
+      wikipediaDescription: `This component is used to get search result from https://www.wikipedia.org/. Typically, it performs as a supplement to knowledgebases. Top N specifies the number of search results you need to adopt.`,
      promptText: `Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:
        {input}
  The above is the content you need to summarize.`,
@ -612,6 +613,7 @@ The above is the content you need to summarize.`,
      messageHistoryWindowSize: 'Message window size',
      messageHistoryWindowSizeTip:
        'The  window size of conversation history that needed to be seen by LLM. The larger the better. But be careful with the maximum content length of LLM.',
+      wikipedia: 'Wikipedia',
    },
    footer: {
      profile: 'All rights reserved @ React',
--- a/web/src/locales/zh-traditional.ts
+++ b/web/src/locales/zh-traditional.ts
@ -541,6 +541,7 @@ export default {
      messageDescription:
        '此元件用於向使用者發送靜態訊息。您可以準備幾條訊息，這些訊息將隨機選擇。',
      keywordDescription: `該組件用於從用戶的問題中提取關鍵字。 Top N指定需要提取的關鍵字數量。`,
+      wikipediaDescription: `此元件用於從 https://www.wikipedia.org/ 取得搜尋結果。通常，它充當知識庫的補充。 Top N 指定您需要採用的搜尋結果的數量。`,
      promptText: `請總結以下段落。注意數字，不要胡編亂造。段落如下：
 {input}
 以上就是你需要總結的內容。`,
@ -573,6 +574,7 @@ export default {
      messageHistoryWindowSize: '歷史訊息視窗大小',
      messageHistoryWindowSizeTip:
        'LLM需要查看的對話記錄的視窗大小。越大越好。但要注意LLM的最大內容長度。',
+      wikipedia: '維基百科',
    },
    footer: {
      profile: '“保留所有權利 @ react”',
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@ -559,6 +559,7 @@ export default {
      messageDescription:
        '此组件用于向用户发送静态信息。您可以准备几条消息，这些消息将被随机选择。',
      keywordDescription: `该组件用于从用户的问题中提取关键词。Top N指定需要提取的关键词数量。`,
+      wikipediaDescription: `此组件用于从 https://www.wikipedia.org/ 获取搜索结果。通常，它作为知识库的补充。Top N 指定您需要采用的搜索结果数量。`,
      promptText: `请总结以下段落。注意数字，不要胡编乱造。段落如下：
 {input}
 以上就是你需要总结的内容。`,
@ -591,6 +592,7 @@ export default {
      messageHistoryWindowSize: '历史消息窗口大小',
      messageHistoryWindowSizeTip:
        'LLM 需要查看的对话历史窗口大小。越大越好。但要注意 LLM 的最大内容长度。',
+      wikipedia: '维基百科',
    },
    footer: {
      profile: 'All rights reserved @ React',
--- a/web/src/pages/flow/constant.tsx
+++ b/web/src/pages/flow/constant.tsx
@ -1,6 +1,7 @@
 import { ReactComponent as BaiduIcon } from '@/assets/svg/baidu.svg';
 import { ReactComponent as DuckIcon } from '@/assets/svg/duck.svg';
 import { ReactComponent as KeywordIcon } from '@/assets/svg/keyword.svg';
+import { ReactComponent as WikipediaIcon } from '@/assets/svg/wikipedia.svg';
 import { variableEnabledFieldMap } from '@/constants/chat';
 import i18n from '@/locales/config';

@ -33,6 +34,7 @@ export enum Operator {
  KeywordExtract = 'KeywordExtract',
  Baidu = 'Baidu',
  DuckDuckGo = 'DuckDuckGo',
+  Wikipedia = 'Wikipedia',
 }

 export const operatorIconMap = {
@ -47,6 +49,7 @@ export const operatorIconMap = {
  [Operator.KeywordExtract]: KeywordIcon,
  [Operator.DuckDuckGo]: DuckIcon,
  [Operator.Baidu]: BaiduIcon,
+  [Operator.Wikipedia]: WikipediaIcon,
 };

 export const operatorMap = {
@ -107,6 +110,9 @@ export const operatorMap = {
    color: '#aea00c',
  },
  [Operator.Baidu]: {},
+  [Operator.Wikipedia]: {
+    backgroundColor: '#dee0e2',
+  },
 };

 export const componentMenuList = [
@ -140,6 +146,9 @@ export const componentMenuList = [
  {
    name: Operator.Baidu,
  },
+  {
+    name: Operator.Wikipedia,
+  },
 ];

 export const initialRetrievalValues = {
@ -207,6 +216,11 @@ export const initialBaiduValues = {
  top_n: 10,
 };

+export const initialWikipediaValues = {
+  top_n: 10,
+  language: 'en',
+};
+
 export const CategorizeAnchorPointPositions = [
  { top: 1, right: 34 },
  { top: 8, right: 18 },
@ -265,6 +279,7 @@ export const RestrictedUpstreamMap = {
  ],
  [Operator.Baidu]: [Operator.Begin, Operator.Retrieval],
  [Operator.DuckDuckGo]: [Operator.Begin, Operator.Retrieval],
+  [Operator.Wikipedia]: [Operator.Begin, Operator.Retrieval],
 };

 export const NodeMap = {
@ -279,4 +294,288 @@ export const NodeMap = {
  [Operator.KeywordExtract]: 'ragNode',
  [Operator.DuckDuckGo]: 'ragNode',
  [Operator.Baidu]: 'ragNode',
+  [Operator.Wikipedia]: 'ragNode',
 };
+
+export const LanguageOptions = [
+  {
+    value: 'af',
+    label: 'Afrikaans',
+  },
+  {
+    value: 'pl',
+    label: 'Polski',
+  },
+  {
+    value: 'ar',
+    label: 'العربية',
+  },
+  {
+    value: 'ast',
+    label: 'Asturianu',
+  },
+  {
+    value: 'az',
+    label: 'Azərbaycanca',
+  },
+  {
+    value: 'bg',
+    label: 'Български',
+  },
+  {
+    value: 'nan',
+    label: '閩南語 / Bân-lâm-gú',
+  },
+  {
+    value: 'bn',
+    label: 'বাংলা',
+  },
+  {
+    value: 'be',
+    label: 'Беларуская',
+  },
+  {
+    value: 'ca',
+    label: 'Català',
+  },
+  {
+    value: 'cs',
+    label: 'Čeština',
+  },
+  {
+    value: 'cy',
+    label: 'Cymraeg',
+  },
+  {
+    value: 'da',
+    label: 'Dansk',
+  },
+  {
+    value: 'de',
+    label: 'Deutsch',
+  },
+  {
+    value: 'et',
+    label: 'Eesti',
+  },
+  {
+    value: 'el',
+    label: 'Ελληνικά',
+  },
+  {
+    value: 'en',
+    label: 'English',
+  },
+  {
+    value: 'es',
+    label: 'Español',
+  },
+  {
+    value: 'eo',
+    label: 'Esperanto',
+  },
+  {
+    value: 'eu',
+    label: 'Euskara',
+  },
+  {
+    value: 'fa',
+    label: 'فارسی',
+  },
+  {
+    value: 'fr',
+    label: 'Français',
+  },
+  {
+    value: 'gl',
+    label: 'Galego',
+  },
+  {
+    value: 'ko',
+    label: '한국어',
+  },
+  {
+    value: 'hy',
+    label: 'Հայերեն',
+  },
+  {
+    value: 'hi',
+    label: 'हिन्दी',
+  },
+  {
+    value: 'hr',
+    label: 'Hrvatski',
+  },
+  {
+    value: 'id',
+    label: 'Bahasa Indonesia',
+  },
+  {
+    value: 'it',
+    label: 'Italiano',
+  },
+  {
+    value: 'he',
+    label: 'עברית',
+  },
+  {
+    value: 'ka',
+    label: 'ქართული',
+  },
+  {
+    value: 'lld',
+    label: 'Ladin',
+  },
+  {
+    value: 'la',
+    label: 'Latina',
+  },
+  {
+    value: 'lv',
+    label: 'Latviešu',
+  },
+  {
+    value: 'lt',
+    label: 'Lietuvių',
+  },
+  {
+    value: 'hu',
+    label: 'Magyar',
+  },
+  {
+    value: 'mk',
+    label: 'Македонски',
+  },
+  {
+    value: 'arz',
+    label: 'مصرى',
+  },
+  {
+    value: 'ms',
+    label: 'Bahasa Melayu',
+  },
+  {
+    value: 'min',
+    label: 'Bahaso Minangkabau',
+  },
+  {
+    value: 'my',
+    label: 'မြန်မာဘာသာ',
+  },
+  {
+    value: 'nl',
+    label: 'Nederlands',
+  },
+  {
+    value: 'ja',
+    label: '日本語',
+  },
+  {
+    value: 'no',
+    label: 'Norsk (bokmål)',
+  },
+  {
+    value: 'nn',
+    label: 'Norsk (nynorsk)',
+  },
+  {
+    value: 'ce',
+    label: 'Нохчийн',
+  },
+  {
+    value: 'uz',
+    label: 'Oʻzbekcha / Ўзбекча',
+  },
+  {
+    value: 'pt',
+    label: 'Português',
+  },
+  {
+    value: 'kk',
+    label: 'Қазақша / Qazaqşa / قازاقشا',
+  },
+  {
+    value: 'ro',
+    label: 'Română',
+  },
+  {
+    value: 'ru',
+    label: 'Русский',
+  },
+  {
+    value: 'ceb',
+    label: 'Sinugboanong Binisaya',
+  },
+  {
+    value: 'sk',
+    label: 'Slovenčina',
+  },
+  {
+    value: 'sl',
+    label: 'Slovenščina',
+  },
+  {
+    value: 'sr',
+    label: 'Српски / Srpski',
+  },
+  {
+    value: 'sh',
+    label: 'Srpskohrvatski / Српскохрватски',
+  },
+  {
+    value: 'fi',
+    label: 'Suomi',
+  },
+  {
+    value: 'sv',
+    label: 'Svenska',
+  },
+  {
+    value: 'ta',
+    label: 'தமிழ்',
+  },
+  {
+    value: 'tt',
+    label: 'Татарча / Tatarça',
+  },
+  {
+    value: 'th',
+    label: 'ภาษาไทย',
+  },
+  {
+    value: 'tg',
+    label: 'Тоҷикӣ',
+  },
+  {
+    value: 'azb',
+    label: 'تۆرکجه',
+  },
+  {
+    value: 'tr',
+    label: 'Türkçe',
+  },
+  {
+    value: 'uk',
+    label: 'Українська',
+  },
+  {
+    value: 'ur',
+    label: 'اردو',
+  },
+  {
+    value: 'vi',
+    label: 'Tiếng Việt',
+  },
+  {
+    value: 'war',
+    label: 'Winaray',
+  },
+  {
+    value: 'zh',
+    label: '中文',
+  },
+  {
+    value: 'yue',
+    label: '粵語',
+  },
+];
--- a/web/src/pages/flow/flow-drawer/index.tsx
+++ b/web/src/pages/flow/flow-drawer/index.tsx
@ -17,6 +17,7 @@ import OperatorIcon from '../operator-icon';
 import RelevantForm from '../relevant-form';
 import RetrievalForm from '../retrieval-form';
 import RewriteQuestionForm from '../rewrite-question-form';
+import WikipediaForm from '../wikipedia-form';

 import styles from './index.less';

@ -36,6 +37,7 @@ const FormMap = {
  [Operator.Baidu]: BaiduForm,
  [Operator.DuckDuckGo]: DuckDuckGoForm,
  [Operator.KeywordExtract]: KeywordExtractForm,
+  [Operator.Wikipedia]: WikipediaForm,
 };

 const EmptyContent = () => <div>empty</div>;
--- a/web/src/pages/flow/hooks.ts
+++ b/web/src/pages/flow/hooks.ts
@ -41,6 +41,7 @@ import {
  initialRelevantValues,
  initialRetrievalValues,
  initialRewriteQuestionValues,
+  initialWikipediaValues,
 } from './constant';
 import { ICategorizeForm, IRelevantForm } from './interface';
 import useGraphStore, { RFState } from './store';
@ -88,6 +89,7 @@ export const useInitializeOperatorParams = () => {
      },
      [Operator.DuckDuckGo]: initialDuckValues,
      [Operator.Baidu]: initialBaiduValues,
+      [Operator.Wikipedia]: initialWikipediaValues,
    };
  }, [llmId]);

--- a/web/src/pages/flow/wikipedia-form/index.tsx
+++ b/web/src/pages/flow/wikipedia-form/index.tsx
@ -0,0 +1,27 @@
+import TopNItem from '@/components/top-n-item';
+import { useTranslate } from '@/hooks/commonHooks';
+import { Form, Select } from 'antd';
+import { LanguageOptions } from '../constant';
+import { IOperatorForm } from '../interface';
+
+const WikipediaForm = ({ onValuesChange, form }: IOperatorForm) => {
+  const { t } = useTranslate('common');
+
+  return (
+    <Form
+      name="basic"
+      labelCol={{ span: 6 }}
+      wrapperCol={{ span: 18 }}
+      autoComplete="off"
+      form={form}
+      onValuesChange={onValuesChange}
+    >
+      <TopNItem initialValue={10}></TopNItem>
+      <Form.Item label={t('language')} name={'language'}>
+        <Select options={LanguageOptions}></Select>
+      </Form.Item>
+    </Form>
+  );
+};
+
+export default WikipediaForm;