From 5a729a69cd78d1a71af6a3f821ef22eda78e8ea3 Mon Sep 17 00:00:00 2001 From: "Leo.Wang" Date: Fri, 16 Aug 2024 16:54:49 +0800 Subject: [PATCH] feat: tools/gitlab (#7329) Co-authored-by: crazywoola <427733928@qq.com> --- .../tools/provider/builtin/gitlab/gitlab.py | 2 +- .../tools/provider/builtin/gitlab/gitlab.yaml | 32 +++---- .../builtin/gitlab/tools/gitlab_commits.py | 34 ++++--- .../builtin/gitlab/tools/gitlab_commits.yaml | 45 ++++++--- .../builtin/gitlab/tools/gitlab_files.py | 95 +++++++++++++++++++ .../builtin/gitlab/tools/gitlab_files.yaml | 45 +++++++++ 6 files changed, 212 insertions(+), 41 deletions(-) create mode 100644 api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py create mode 100644 api/core/tools/provider/builtin/gitlab/tools/gitlab_files.yaml diff --git a/api/core/tools/provider/builtin/gitlab/gitlab.py b/api/core/tools/provider/builtin/gitlab/gitlab.py index fca34ae15f..0c13ec662a 100644 --- a/api/core/tools/provider/builtin/gitlab/gitlab.py +++ b/api/core/tools/provider/builtin/gitlab/gitlab.py @@ -29,6 +29,6 @@ class GitlabProvider(BuiltinToolProviderController): if response.status_code != 200: raise ToolProviderCredentialValidationError((response.json()).get('message')) except Exception as e: - raise ToolProviderCredentialValidationError("Gitlab Access Tokens and Api Version is invalid. {}".format(e)) + raise ToolProviderCredentialValidationError("Gitlab Access Tokens is invalid. {}".format(e)) except Exception as e: raise ToolProviderCredentialValidationError(str(e)) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/gitlab/gitlab.yaml b/api/core/tools/provider/builtin/gitlab/gitlab.yaml index b5feea2382..22d7ebf73a 100644 --- a/api/core/tools/provider/builtin/gitlab/gitlab.yaml +++ b/api/core/tools/provider/builtin/gitlab/gitlab.yaml @@ -2,37 +2,37 @@ identity: author: Leo.Wang name: gitlab label: - en_US: Gitlab - zh_Hans: Gitlab + en_US: GitLab + zh_Hans: GitLab description: - en_US: Gitlab plugin for commit - zh_Hans: 用于获取Gitlab commit的插件 + en_US: GitLab plugin, API v4 only. + zh_Hans: 用于获取GitLab内容的插件，目前仅支持 API v4。 icon: gitlab.svg credentials_for_provider: access_tokens: type: secret-input required: true label: - en_US: Gitlab access token - zh_Hans: Gitlab access token + en_US: GitLab access token + zh_Hans: GitLab access token placeholder: - en_US: Please input your Gitlab access token - zh_Hans: 请输入你的 Gitlab access token + en_US: Please input your GitLab access token + zh_Hans: 请输入你的 GitLab access token help: - en_US: Get your Gitlab access token from Gitlab - zh_Hans: 从 Gitlab 获取您的 access token + en_US: Get your GitLab access token from GitLab + zh_Hans: 从 GitLab 获取您的 access token url: https://docs.gitlab.com/16.9/ee/api/oauth2.html site_url: type: text-input required: false default: 'https://gitlab.com' label: - en_US: Gitlab site url - zh_Hans: Gitlab site url + en_US: GitLab site url + zh_Hans: GitLab site url placeholder: - en_US: Please input your Gitlab site url - zh_Hans: 请输入你的 Gitlab site url + en_US: Please input your GitLab site url + zh_Hans: 请输入你的 GitLab site url help: - en_US: Find your Gitlab url - zh_Hans: 找到你的Gitlab url + en_US: Find your GitLab url + zh_Hans: 找到你的 GitLab url url: https://gitlab.com/help diff --git a/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py b/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py index 212bdb03ab..880d722bda 100644 --- a/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py +++ b/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py @@ -18,6 +18,7 @@ class GitlabCommitsTool(BuiltinTool): employee = tool_parameters.get('employee', '') start_time = tool_parameters.get('start_time', '') end_time = tool_parameters.get('end_time', '') + change_type = tool_parameters.get('change_type', 'all') if not project: return self.create_text_message('Project is required') @@ -36,11 +37,11 @@ class GitlabCommitsTool(BuiltinTool): site_url = 'https://gitlab.com' # Get commit content - result = self.fetch(user_id, site_url, access_token, project, employee, start_time, end_time) + result = self.fetch(user_id, site_url, access_token, project, employee, start_time, end_time, change_type) - return self.create_text_message(json.dumps(result, ensure_ascii=False)) + return [self.create_json_message(item) for item in result] - def fetch(self,user_id: str, site_url: str, access_token: str, project: str, employee: str = None, start_time: str = '', end_time: str = '') -> list[dict[str, Any]]: + def fetch(self,user_id: str, site_url: str, access_token: str, project: str, employee: str = None, start_time: str = '', end_time: str = '', change_type: str = '') -> list[dict[str, Any]]: domain = site_url headers = {"PRIVATE-TOKEN": access_token} results = [] @@ -74,7 +75,7 @@ class GitlabCommitsTool(BuiltinTool): for commit in commits: commit_sha = commit['id'] - print(f"\tCommit SHA: {commit_sha}") + author_name = commit['author_name'] diff_url = f"{domain}/api/v4/projects/{project_id}/repository/commits/{commit_sha}/diff" diff_response = requests.get(diff_url, headers=headers) @@ -87,14 +88,23 @@ class GitlabCommitsTool(BuiltinTool): removed_lines = diff['diff'].count('\n-') total_changes = added_lines + removed_lines - if total_changes > 1: - final_code = ''.join([line[1:] for line in diff['diff'].split('\n') if line.startswith('+') and not line.startswith('+++')]) - results.append({ - "project": project_name, - "commit_sha": commit_sha, - "diff": final_code - }) - print(f"Commit code:{final_code}") + if change_type == "new": + if added_lines > 1: + final_code = ''.join([line[1:] for line in diff['diff'].split('\n') if line.startswith('+') and not line.startswith('+++')]) + results.append({ + "commit_sha": commit_sha, + "author_name": author_name, + "diff": final_code + }) + else: + if total_changes > 1: + final_code = ''.join([line[1:] for line in diff['diff'].split('\n') if (line.startswith('+') or line.startswith('-')) and not line.startswith('+++') and not line.startswith('---')]) + final_code_escaped = json.dumps(final_code)[1:-1] # Escape the final code + results.append({ + "commit_sha": commit_sha, + "author_name": author_name, + "diff": final_code_escaped + }) except requests.RequestException as e: print(f"Error fetching data from GitLab: {e}") diff --git a/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml b/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml index fc4e7eb7bb..dd4e31d663 100644 --- a/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml +++ b/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml @@ -2,24 +2,24 @@ identity: name: gitlab_commits author: Leo.Wang label: - en_US: Gitlab Commits - zh_Hans: Gitlab代码提交内容 + en_US: GitLab Commits + zh_Hans: GitLab 提交内容查询 description: human: - en_US: A tool for query gitlab commits. Input should be a exists username. - zh_Hans: 一个用于查询gitlab代码提交记录的的工具，输入的内容应该是一个已存在的用户名或者项目名。 - llm: A tool for query gitlab commits. Input should be a exists username or project. + en_US: A tool for query GitLab commits, Input should be a exists username or projec. + zh_Hans: 一个用于查询 GitLab 代码提交内容的工具，输入的内容应该是一个已存在的用户名或者项目名。 + llm: A tool for query GitLab commits, Input should be a exists username or project. parameters: - - name: employee + - name: username type: string required: false label: - en_US: employee + en_US: username zh_Hans: 员工用户名 human_description: - en_US: employee + en_US: username zh_Hans: 员工用户名 - llm_description: employee for gitlab + llm_description: User name for GitLab form: llm - name: project type: string @@ -30,7 +30,7 @@ parameters: human_description: en_US: project zh_Hans: 项目名 - llm_description: project for gitlab + llm_description: project for GitLab form: llm - name: start_time type: string @@ -41,7 +41,7 @@ parameters: human_description: en_US: start_time zh_Hans: 开始时间 - llm_description: start_time for gitlab + llm_description: Start time for GitLab form: llm - name: end_time type: string @@ -52,5 +52,26 @@ parameters: human_description: en_US: end_time zh_Hans: 结束时间 - llm_description: end_time for gitlab + llm_description: End time for GitLab + form: llm + - name: change_type + type: select + required: false + options: + - value: all + label: + en_US: all + zh_Hans: 所有 + - value: new + label: + en_US: new + zh_Hans: 新增 + default: all + label: + en_US: change_type + zh_Hans: 变更类型 + human_description: + en_US: change_type + zh_Hans: 变更类型 + llm_description: Content change type for GitLab form: llm diff --git a/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py b/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py new file mode 100644 index 0000000000..7fa1d0d112 --- /dev/null +++ b/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py @@ -0,0 +1,95 @@ +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GitlabFilesTool(BuiltinTool): + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + project = tool_parameters.get('project', '') + branch = tool_parameters.get('branch', '') + path = tool_parameters.get('path', '') + + + if not project: + return self.create_text_message('Project is required') + if not branch: + return self.create_text_message('Branch is required') + + if not path: + return self.create_text_message('Path is required') + + access_token = self.runtime.credentials.get('access_tokens') + site_url = self.runtime.credentials.get('site_url') + + if 'access_tokens' not in self.runtime.credentials or not self.runtime.credentials.get('access_tokens'): + return self.create_text_message("Gitlab API Access Tokens is required.") + if 'site_url' not in self.runtime.credentials or not self.runtime.credentials.get('site_url'): + site_url = 'https://gitlab.com' + + # Get project ID from project name + project_id = self.get_project_id(site_url, access_token, project) + if not project_id: + return self.create_text_message(f"Project '{project}' not found.") + + # Get commit content + result = self.fetch(user_id, project_id, site_url, access_token, branch, path) + + return [self.create_json_message(item) for item in result] + + def extract_project_name_and_path(self, path: str) -> tuple[str, str]: + parts = path.split('/', 1) + if len(parts) < 2: + return None, None + return parts[0], parts[1] + + def get_project_id(self, site_url: str, access_token: str, project_name: str) -> Union[str, None]: + headers = {"PRIVATE-TOKEN": access_token} + try: + url = f"{site_url}/api/v4/projects?search={project_name}" + response = requests.get(url, headers=headers) + response.raise_for_status() + projects = response.json() + for project in projects: + if project['name'] == project_name: + return project['id'] + except requests.RequestException as e: + print(f"Error fetching project ID from GitLab: {e}") + return None + + def fetch(self,user_id: str, project_id: str, site_url: str, access_token: str, branch: str, path: str = None) -> list[dict[str, Any]]: + domain = site_url + headers = {"PRIVATE-TOKEN": access_token} + results = [] + + try: + # List files and directories in the given path + url = f"{domain}/api/v4/projects/{project_id}/repository/tree?path={path}&ref={branch}" + response = requests.get(url, headers=headers) + response.raise_for_status() + items = response.json() + + for item in items: + item_path = item['path'] + if item['type'] == 'tree': # It's a directory + results.extend(self.fetch(project_id, site_url, access_token, branch, item_path)) + else: # It's a file + file_url = f"{domain}/api/v4/projects/{project_id}/repository/files/{item_path}/raw?ref={branch}" + file_response = requests.get(file_url, headers=headers) + file_response.raise_for_status() + file_content = file_response.text + results.append({ + "path": item_path, + "branch": branch, + "content": file_content + }) + except requests.RequestException as e: + print(f"Error fetching data from GitLab: {e}") + + return results \ No newline at end of file diff --git a/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.yaml b/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.yaml new file mode 100644 index 0000000000..d99b6254c1 --- /dev/null +++ b/api/core/tools/provider/builtin/gitlab/tools/gitlab_files.yaml @@ -0,0 +1,45 @@ +identity: + name: gitlab_files + author: Leo.Wang + label: + en_US: GitLab Files + zh_Hans: GitLab 文件获取 +description: + human: + en_US: A tool for query GitLab files, Input should be branch and a exists file or directory path. + zh_Hans: 一个用于查询 GitLab 文件的工具，输入的内容应该是分支和一个已存在文件或者文件夹路径。 + llm: A tool for query GitLab files, Input should be a exists file or directory path. +parameters: + - name: project + type: string + required: true + label: + en_US: project + zh_Hans: 项目 + human_description: + en_US: project + zh_Hans: 项目 + llm_description: Project for GitLab + form: llm + - name: branch + type: string + required: true + label: + en_US: branch + zh_Hans: 分支 + human_description: + en_US: branch + zh_Hans: 分支 + llm_description: Branch for GitLab + form: llm + - name: path + type: string + required: true + label: + en_US: path + zh_Hans: 文件路径 + human_description: + en_US: path + zh_Hans: 文件路径 + llm_description: File path for GitLab + form: llm