balibabu
commited on
Commit
·
7b030d6
1
Parent(s):
1f75d02
feat: After selecting the parsing method as knowledge graph, the delimiter and chunk token number are displayed. #1594 (#1929)
Browse files### What problem does this PR solve?
feat: After selecting the parsing method as knowledge graph, the
delimiter and chunk token number are displayed. #1594
### Type of change
- [x] New Feature (non-breaking change which adds functionality)
- web/src/components/chunk-method-modal/hooks.ts +5 -1
- web/src/components/chunk-method-modal/index.tsx +8 -4
- web/src/components/edit-tag/index.less +5 -0
- web/src/components/max-token-number.tsx +10 -4
- web/src/hooks/logic-hooks.ts +23 -0
- web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx +12 -2
- web/src/pages/add-knowledge/components/knowledge-setting/hooks.ts +4 -0
web/src/components/chunk-method-modal/hooks.ts
CHANGED
@@ -1,4 +1,6 @@
|
|
|
|
1 |
import { useSelectParserList } from '@/hooks/user-setting-hooks';
|
|
|
2 |
import { useEffect, useMemo, useState } from 'react';
|
3 |
|
4 |
const ParserListMap = new Map([
|
@@ -84,9 +86,11 @@ export const useFetchParserListOnMount = (
|
|
84 |
documentId: string,
|
85 |
parserId: string,
|
86 |
documentExtension: string,
|
|
|
87 |
) => {
|
88 |
const [selectedTag, setSelectedTag] = useState('');
|
89 |
const parserList = useSelectParserList();
|
|
|
90 |
|
91 |
const nextParserList = useMemo(() => {
|
92 |
const key = [...ParserListMap.keys()].find((x) =>
|
@@ -108,7 +112,7 @@ export const useFetchParserListOnMount = (
|
|
108 |
}, [parserId, documentId]);
|
109 |
|
110 |
const handleChange = (tag: string) => {
|
111 |
-
|
112 |
setSelectedTag(tag);
|
113 |
};
|
114 |
|
|
|
1 |
+
import { useHandleChunkMethodSelectChange } from '@/hooks/logic-hooks';
|
2 |
import { useSelectParserList } from '@/hooks/user-setting-hooks';
|
3 |
+
import { FormInstance } from 'antd';
|
4 |
import { useEffect, useMemo, useState } from 'react';
|
5 |
|
6 |
const ParserListMap = new Map([
|
|
|
86 |
documentId: string,
|
87 |
parserId: string,
|
88 |
documentExtension: string,
|
89 |
+
form: FormInstance,
|
90 |
) => {
|
91 |
const [selectedTag, setSelectedTag] = useState('');
|
92 |
const parserList = useSelectParserList();
|
93 |
+
const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form);
|
94 |
|
95 |
const nextParserList = useMemo(() => {
|
96 |
const key = [...ParserListMap.keys()].find((x) =>
|
|
|
112 |
}, [parserId, documentId]);
|
113 |
|
114 |
const handleChange = (tag: string) => {
|
115 |
+
handleChunkMethodSelectChange(tag);
|
116 |
setSelectedTag(tag);
|
117 |
};
|
118 |
|
web/src/components/chunk-method-modal/index.tsx
CHANGED
@@ -62,12 +62,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|
62 |
parserConfig,
|
63 |
loading,
|
64 |
}) => {
|
|
|
65 |
const { parserList, handleChange, selectedTag } = useFetchParserListOnMount(
|
66 |
documentId,
|
67 |
parserId,
|
68 |
documentExtension,
|
|
|
69 |
);
|
70 |
-
const [form] = Form.useForm();
|
71 |
const { t } = useTranslate('knowledgeDetails');
|
72 |
|
73 |
const handleOk = async () => {
|
@@ -89,12 +90,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|
89 |
return (
|
90 |
isPdf &&
|
91 |
hidePagesChunkMethods
|
92 |
-
.filter((x) => x !== 'one'
|
93 |
.every((x) => x !== selectedTag)
|
94 |
);
|
95 |
}, [selectedTag, isPdf]);
|
96 |
|
97 |
-
const showMaxTokenNumber =
|
|
|
98 |
|
99 |
const hideDivider = [showPages, showOne, showMaxTokenNumber].every(
|
100 |
(x) => x === false,
|
@@ -271,7 +273,9 @@ const ChunkMethodModal: React.FC<IProps> = ({
|
|
271 |
)}
|
272 |
{showMaxTokenNumber && (
|
273 |
<>
|
274 |
-
<MaxTokenNumber
|
|
|
|
|
275 |
<Delimiter></Delimiter>
|
276 |
</>
|
277 |
)}
|
|
|
62 |
parserConfig,
|
63 |
loading,
|
64 |
}) => {
|
65 |
+
const [form] = Form.useForm();
|
66 |
const { parserList, handleChange, selectedTag } = useFetchParserListOnMount(
|
67 |
documentId,
|
68 |
parserId,
|
69 |
documentExtension,
|
70 |
+
form,
|
71 |
);
|
|
|
72 |
const { t } = useTranslate('knowledgeDetails');
|
73 |
|
74 |
const handleOk = async () => {
|
|
|
90 |
return (
|
91 |
isPdf &&
|
92 |
hidePagesChunkMethods
|
93 |
+
.filter((x) => x !== 'one')
|
94 |
.every((x) => x !== selectedTag)
|
95 |
);
|
96 |
}, [selectedTag, isPdf]);
|
97 |
|
98 |
+
const showMaxTokenNumber =
|
99 |
+
selectedTag === 'naive' || selectedTag === 'knowledge_graph';
|
100 |
|
101 |
const hideDivider = [showPages, showOne, showMaxTokenNumber].every(
|
102 |
(x) => x === false,
|
|
|
273 |
)}
|
274 |
{showMaxTokenNumber && (
|
275 |
<>
|
276 |
+
<MaxTokenNumber
|
277 |
+
max={selectedTag === 'knowledge_graph' ? 8192 * 2 : 2048}
|
278 |
+
></MaxTokenNumber>
|
279 |
<Delimiter></Delimiter>
|
280 |
</>
|
281 |
)}
|
web/src/components/edit-tag/index.less
CHANGED
@@ -1,3 +1,8 @@
|
|
1 |
.tweenGroup {
|
2 |
display: inline-block;
|
|
|
|
|
|
|
|
|
|
|
3 |
}
|
|
|
1 |
.tweenGroup {
|
2 |
display: inline-block;
|
3 |
+
:global(.ant-tag) {
|
4 |
+
margin-bottom: 8px;
|
5 |
+
font-size: 14px;
|
6 |
+
padding: 2px 8px;
|
7 |
+
}
|
8 |
}
|
web/src/components/max-token-number.tsx
CHANGED
@@ -1,7 +1,12 @@
|
|
1 |
import { useTranslate } from '@/hooks/common-hooks';
|
2 |
import { Flex, Form, InputNumber, Slider } from 'antd';
|
3 |
|
4 |
-
|
|
|
|
|
|
|
|
|
|
|
5 |
const { t } = useTranslate('knowledgeConfiguration');
|
6 |
|
7 |
return (
|
@@ -11,18 +16,19 @@ const MaxTokenNumber = () => {
|
|
11 |
<Form.Item
|
12 |
name={['parser_config', 'chunk_token_num']}
|
13 |
noStyle
|
14 |
-
initialValue={
|
15 |
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
|
16 |
>
|
17 |
-
<Slider max={
|
18 |
</Form.Item>
|
19 |
</Flex>
|
20 |
<Form.Item
|
21 |
name={['parser_config', 'chunk_token_num']}
|
22 |
noStyle
|
|
|
23 |
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
|
24 |
>
|
25 |
-
<InputNumber max={
|
26 |
</Form.Item>
|
27 |
</Flex>
|
28 |
</Form.Item>
|
|
|
1 |
import { useTranslate } from '@/hooks/common-hooks';
|
2 |
import { Flex, Form, InputNumber, Slider } from 'antd';
|
3 |
|
4 |
+
interface IProps {
|
5 |
+
initialValue?: number;
|
6 |
+
max?: number;
|
7 |
+
}
|
8 |
+
|
9 |
+
const MaxTokenNumber = ({ initialValue = 128, max = 2048 }: IProps) => {
|
10 |
const { t } = useTranslate('knowledgeConfiguration');
|
11 |
|
12 |
return (
|
|
|
16 |
<Form.Item
|
17 |
name={['parser_config', 'chunk_token_num']}
|
18 |
noStyle
|
19 |
+
initialValue={initialValue}
|
20 |
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
|
21 |
>
|
22 |
+
<Slider max={max} style={{ width: '100%' }} />
|
23 |
</Form.Item>
|
24 |
</Flex>
|
25 |
<Form.Item
|
26 |
name={['parser_config', 'chunk_token_num']}
|
27 |
noStyle
|
28 |
+
initialValue={initialValue}
|
29 |
rules={[{ required: true, message: t('chunkTokenNumberMessage') }]}
|
30 |
>
|
31 |
+
<InputNumber max={max} min={0} />
|
32 |
</Form.Item>
|
33 |
</Flex>
|
34 |
</Form.Item>
|
web/src/hooks/logic-hooks.ts
CHANGED
@@ -8,6 +8,7 @@ import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
|
|
8 |
import api from '@/utils/api';
|
9 |
import { getAuthorization } from '@/utils/authorization-util';
|
10 |
import { PaginationProps } from 'antd';
|
|
|
11 |
import axios from 'axios';
|
12 |
import { EventSourceParserStream } from 'eventsource-parser/stream';
|
13 |
import {
|
@@ -337,3 +338,25 @@ export const useFetchModelId = () => {
|
|
337 |
|
338 |
return tenantInfo?.llm_id ?? '';
|
339 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
import api from '@/utils/api';
|
9 |
import { getAuthorization } from '@/utils/authorization-util';
|
10 |
import { PaginationProps } from 'antd';
|
11 |
+
import { FormInstance } from 'antd/lib';
|
12 |
import axios from 'axios';
|
13 |
import { EventSourceParserStream } from 'eventsource-parser/stream';
|
14 |
import {
|
|
|
338 |
|
339 |
return tenantInfo?.llm_id ?? '';
|
340 |
};
|
341 |
+
|
342 |
+
const ChunkTokenNumMap = {
|
343 |
+
naive: 128,
|
344 |
+
knowledge_graph: 8192,
|
345 |
+
};
|
346 |
+
|
347 |
+
export const useHandleChunkMethodSelectChange = (form: FormInstance) => {
|
348 |
+
// const form = Form.useFormInstance();
|
349 |
+
const handleChange = useCallback(
|
350 |
+
(value: string) => {
|
351 |
+
if (value in ChunkTokenNumMap) {
|
352 |
+
form.setFieldValue(
|
353 |
+
['parser_config', 'chunk_token_num'],
|
354 |
+
ChunkTokenNumMap[value as keyof typeof ChunkTokenNumMap],
|
355 |
+
);
|
356 |
+
}
|
357 |
+
},
|
358 |
+
[form],
|
359 |
+
);
|
360 |
+
|
361 |
+
return handleChange;
|
362 |
+
};
|
web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx
CHANGED
@@ -6,6 +6,7 @@ import ParseConfiguration, {
|
|
6 |
showRaptorParseConfiguration,
|
7 |
} from '@/components/parse-configuration';
|
8 |
import { useTranslate } from '@/hooks/common-hooks';
|
|
|
9 |
import { normFile } from '@/utils/file-util';
|
10 |
import { PlusOutlined } from '@ant-design/icons';
|
11 |
import { Button, Form, Input, Radio, Select, Space, Upload } from 'antd';
|
@@ -24,6 +25,7 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
|
|
24 |
const { parserList, embeddingModelOptions, disabled } =
|
25 |
useFetchKnowledgeConfigurationOnMount(form);
|
26 |
const { t } = useTranslate('knowledgeConfiguration');
|
|
|
27 |
|
28 |
return (
|
29 |
<Form form={form} name="validateOnly" layout="vertical" autoComplete="off">
|
@@ -91,7 +93,11 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
|
|
91 |
tooltip={t('chunkMethodTip')}
|
92 |
rules={[{ required: true }]}
|
93 |
>
|
94 |
-
<Select
|
|
|
|
|
|
|
|
|
95 |
{parserList.map((x) => (
|
96 |
<Option value={x.value} key={x.value}>
|
97 |
{x.label}
|
@@ -107,7 +113,11 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
|
|
107 |
return (
|
108 |
<>
|
109 |
{parserId === 'knowledge_graph' && (
|
110 |
-
|
|
|
|
|
|
|
|
|
111 |
)}
|
112 |
{parserId === 'naive' && (
|
113 |
<>
|
|
|
6 |
showRaptorParseConfiguration,
|
7 |
} from '@/components/parse-configuration';
|
8 |
import { useTranslate } from '@/hooks/common-hooks';
|
9 |
+
import { useHandleChunkMethodSelectChange } from '@/hooks/logic-hooks';
|
10 |
import { normFile } from '@/utils/file-util';
|
11 |
import { PlusOutlined } from '@ant-design/icons';
|
12 |
import { Button, Form, Input, Radio, Select, Space, Upload } from 'antd';
|
|
|
25 |
const { parserList, embeddingModelOptions, disabled } =
|
26 |
useFetchKnowledgeConfigurationOnMount(form);
|
27 |
const { t } = useTranslate('knowledgeConfiguration');
|
28 |
+
const handleChunkMethodSelectChange = useHandleChunkMethodSelectChange(form);
|
29 |
|
30 |
return (
|
31 |
<Form form={form} name="validateOnly" layout="vertical" autoComplete="off">
|
|
|
93 |
tooltip={t('chunkMethodTip')}
|
94 |
rules={[{ required: true }]}
|
95 |
>
|
96 |
+
<Select
|
97 |
+
placeholder={t('chunkMethodPlaceholder')}
|
98 |
+
disabled={disabled}
|
99 |
+
onChange={handleChunkMethodSelectChange}
|
100 |
+
>
|
101 |
{parserList.map((x) => (
|
102 |
<Option value={x.value} key={x.value}>
|
103 |
{x.label}
|
|
|
113 |
return (
|
114 |
<>
|
115 |
{parserId === 'knowledge_graph' && (
|
116 |
+
<>
|
117 |
+
<EntityTypesItem></EntityTypesItem>
|
118 |
+
<MaxTokenNumber max={8192 * 2}></MaxTokenNumber>
|
119 |
+
<Delimiter></Delimiter>
|
120 |
+
</>
|
121 |
)}
|
122 |
{parserId === 'naive' && (
|
123 |
<>
|
web/src/pages/add-knowledge/components/knowledge-setting/hooks.ts
CHANGED
@@ -80,5 +80,9 @@ export const useHandleChunkMethodChange = () => {
|
|
80 |
const [form] = Form.useForm();
|
81 |
const chunkMethod = Form.useWatch('parser_id', form);
|
82 |
|
|
|
|
|
|
|
|
|
83 |
return { form, chunkMethod };
|
84 |
};
|
|
|
80 |
const [form] = Form.useForm();
|
81 |
const chunkMethod = Form.useWatch('parser_id', form);
|
82 |
|
83 |
+
useEffect(() => {
|
84 |
+
console.log('🚀 ~ useHandleChunkMethodChange ~ chunkMethod:', chunkMethod);
|
85 |
+
}, [chunkMethod]);
|
86 |
+
|
87 |
return { form, chunkMethod };
|
88 |
};
|