diff --git a/api/core/data_loader/loader/excel.py b/api/core/data_loader/loader/excel.py index 08c0462728..202068432a 100644 --- a/api/core/data_loader/loader/excel.py +++ b/api/core/data_loader/loader/excel.py @@ -39,6 +39,7 @@ class ExcelLoader(BaseLoader): row_dict = dict(zip(keys, list(map(str, row)))) row_dict = {k: v for k, v in row_dict.items() if v} item = ''.join(f'{k}:{v}\n' for k, v in row_dict.items()) - data.append(item) + document = Document(page_content=item) + data.append(document) - return [Document(page_content='\n\n'.join(data))] + return data diff --git a/api/core/data_loader/loader/notion.py b/api/core/data_loader/loader/notion.py index 8235c58cf2..e54266f4ce 100644 --- a/api/core/data_loader/loader/notion.py +++ b/api/core/data_loader/loader/notion.py @@ -81,8 +81,8 @@ class NotionLoader(BaseLoader): docs = [] if notion_page_type == 'database': # get all the pages in the database - page_text = self._get_notion_database_data(notion_obj_id) - docs.append(Document(page_content=page_text)) + page_text_documents = self._get_notion_database_data(notion_obj_id) + docs.extend(page_text_documents) elif notion_page_type == 'page': page_text_list = self._get_notion_block_data(notion_obj_id) for page_text in page_text_list: @@ -94,7 +94,7 @@ class NotionLoader(BaseLoader): def _get_notion_database_data( self, database_id: str, query_dict: Dict[str, Any] = {} - ) -> str: + ) -> List[Document]: """Get all the pages from a Notion database.""" res = requests.post( DATABASE_URL_TMPL.format(database_id=database_id), @@ -110,7 +110,7 @@ class NotionLoader(BaseLoader): database_content_list = [] if 'results' not in data or data["results"] is None: - return "" + return [] for result in data["results"]: properties = result['properties'] data = {} @@ -143,10 +143,10 @@ class NotionLoader(BaseLoader): row_content = row_content + f'{key}:{value_content}\n' else: row_content = row_content + f'{key}:{value}\n' - database_content_list.append(row_content) - database_content_list.append(json.dumps(data, ensure_ascii=False)) + document = Document(page_content=row_content) + database_content_list.append(document) - return "\n\n".join(database_content_list) + return database_content_list def _get_notion_block_data(self, page_id: str) -> List[str]: result_lines_arr = []