AI/dify
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/langgenius/dify.git synced 2025-08-12 04:39:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix/json format (#466)

Browse Source
This commit is contained in:
Jyong 2023-06-28 13:58:50 +08:00 committed by GitHub
parent 2eea114ac0
commit 334f46d0b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/core/data_loader/loader/excel.py
View File

@ -39,6 +39,7 @@ class ExcelLoader(BaseLoader):
row_dict = dict(zip(keys, list(map(str, row)))) row_dict = dict(zip(keys, list(map(str, row))))
row_dict = {k: v for k, v in row_dict.items() if v} row_dict = {k: v for k, v in row_dict.items() if v}
item = ''.join(f'{k}:{v}\n' for k, v in row_dict.items()) item = ''.join(f'{k}:{v}\n' for k, v in row_dict.items())
data.append(item) document = Document(page_content=item)
data.append(document)
return [Document(page_content='\n\n'.join(data))] return data

14
api/core/data_loader/loader/notion.py
View File

@ -81,8 +81,8 @@ class NotionLoader(BaseLoader):
docs = [] docs = []
if notion_page_type == 'database': if notion_page_type == 'database':
# get all the pages in the database # get all the pages in the database
page_text = self._get_notion_database_data(notion_obj_id) page_text_documents = self._get_notion_database_data(notion_obj_id)
docs.append(Document(page_content=page_text)) docs.extend(page_text_documents)
elif notion_page_type == 'page': elif notion_page_type == 'page':
page_text_list = self._get_notion_block_data(notion_obj_id) page_text_list = self._get_notion_block_data(notion_obj_id)
for page_text in page_text_list: for page_text in page_text_list:
@ -94,7 +94,7 @@ class NotionLoader(BaseLoader):
def _get_notion_database_data( def _get_notion_database_data(
self, database_id: str, query_dict: Dict[str, Any] = {} self, database_id: str, query_dict: Dict[str, Any] = {}
) -> str: ) -> List[Document]:
"""Get all the pages from a Notion database.""" """Get all the pages from a Notion database."""
res = requests.post( res = requests.post(
DATABASE_URL_TMPL.format(database_id=database_id), DATABASE_URL_TMPL.format(database_id=database_id),
@ -110,7 +110,7 @@ class NotionLoader(BaseLoader):
database_content_list = [] database_content_list = []
if 'results' not in data or data["results"] is None: if 'results' not in data or data["results"] is None:
return "" return []
for result in data["results"]: for result in data["results"]:
properties = result['properties'] properties = result['properties']
data = {} data = {}
@ -143,10 +143,10 @@ class NotionLoader(BaseLoader):
row_content = row_content + f'{key}:{value_content}\n' row_content = row_content + f'{key}:{value_content}\n'
else: else:
row_content = row_content + f'{key}:{value}\n' row_content = row_content + f'{key}:{value}\n'
database_content_list.append(row_content) document = Document(page_content=row_content)
database_content_list.append(json.dumps(data, ensure_ascii=False)) database_content_list.append(document)
return "\n\n".join(database_content_list) return database_content_list
def _get_notion_block_data(self, page_id: str) -> List[str]: def _get_notion_block_data(self, page_id: str) -> List[str]:
result_lines_arr = [] result_lines_arr = []