diff --git a/scripts/git_commit_mrs_and_issues.py b/scripts/git_commit_mrs_and_issues.py new file mode 100644 index 000000000..854ec390e --- /dev/null +++ b/scripts/git_commit_mrs_and_issues.py @@ -0,0 +1,128 @@ +"""Search for MRs and issues related to a list of commits.""" + +import argparse +import json +import sys +import subprocess +import re + + +def find_cherry_pick_source(commit_hash: str): + """ + For a given commit hash, find the original commit it was cherry-picked from. + + Args: + commit_hash: The commit hash to inspect. + + Returns: + The full hash of the original commit if found, otherwise None. + """ + try: + # Use 'git show' to get the full commit message for the given hash. + # The '-s' flag suppresses the diff output. + # The '--format=%B' flag prints only the raw commit body/message. + commit_message = subprocess.check_output( + ["git", "show", "-s", "--format=%B", commit_hash.strip()], + text=True, + stderr=subprocess.PIPE, + ).strip() + + # This regex looks for the specific line Git adds during a cherry-pick. + # It captures the full 40-character SHA-1 hash. + cherry_pick_pattern = re.compile( + r"\(cherry picked from commit ([a-f0-9]{40})\)" + ) + + # Search the entire commit message for the pattern. + match = cherry_pick_pattern.search(commit_message) + + if match: + # If a match is found, return the captured group (the original commit hash). + return match.group(1) + else: + return None + + except subprocess.CalledProcessError as e: + # This error occurs if the git command fails, e.g., for an invalid hash. + print( + f"Error processing commit '{commit_hash.strip()}': {e.stderr.strip()}", + file=sys.stderr, + ) + return None + except FileNotFoundError: + # This error occurs if the 'git' command itself isn't found. + print( + "Error: 'git' command not found. Please ensure Git is installed and in your PATH.", + file=sys.stderr, + ) + sys.exit(1) + + +def main(): + """ + Main function to read commit hashes from stdin and process them. + """ + parser = argparse.ArgumentParser( + description="A script to download all MRs from GitLab matching specified criteria." + ) + parser.add_argument( + "--merge_requests_file", + type=str, + required=True, + help="JSON file containing all the merge request information extracted via the GitLab API.", + ) + + # E.g. git log --pretty=%H 3e819d83bf52abda16bb53565f6801df40d071f1..3.4.1 + parser.add_argument( + "--commits", + required=True, + help="List of commits, '-' for stdin.", + ) + args = parser.parse_args() + + mrs = [] + with open(args.merge_requests_file, "r") as file: + mrs = json.load(file) + mrs_by_commit = {} + + if args.commits == "-": + commit_hashes = sys.stdin.readlines() + else: + with open(args.commits, "r") as file: + commit_hashes = file.readlines() + + # Arrange commits by SHA. + for mr in mrs: + for key in ["sha", "merge_commit_sha", "squash_commit_sha"]: + sha = mr[key] + if sha: + mrs_by_commit[sha] = mr + + # Find the MRs and issues related to each commit. + info = {} + for sha in commit_hashes: + sha = sha.strip() + if not sha: + continue + + # If a cherry-pick, extract the original hash. + sha = find_cherry_pick_source(sha) or sha + mr = mrs_by_commit.get(sha) + + commit_info = {} + if mr: + commit_info["merge_request"] = mr["iid"] + commit_info["related_issues"] = [ + issue["iid"] for issue in mr["related_issues"] + ] + commit_info["closes_issues"] = [ + issue["iid"] for issue in mr["closes_issues"] + ] + + info[sha] = commit_info + + print(json.dumps(info, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/gitlab_api_deploy_package.py b/scripts/gitlab_api_deploy_package.py new file mode 100644 index 000000000..70dc03ff1 --- /dev/null +++ b/scripts/gitlab_api_deploy_package.py @@ -0,0 +1,136 @@ +"""Helper script to download source archives and upload them to the Eigen GitLab generic package registry.""" + +import os +import requests +import hashlib +import argparse +import sys +import tempfile + +EIGEN_PROJECT_ID = 15462818 # Taken from the gitlab project page. + + +def calculate_sha256(filepath: str): + """Calculates the SHA256 checksum of a file.""" + sha256_hash = hashlib.sha256() + with open(filepath, "rb") as f: + # Read and update hash in chunks of 4K + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + + +def upload_to_generic_registry( + gitlab_private_token: str, package_name: str, package_version: str, filepath: str +): + """Uploads a file to the GitLab generic package registry.""" + headers = {"PRIVATE-TOKEN": gitlab_private_token} + filename = os.path.basename(filepath) + upload_url = f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/packages/generic/{package_name}/{package_version}/{filename}" + + print(f"Uploading {filename} to {upload_url}...") + try: + with open(filepath, "rb") as f: + response = requests.put(upload_url, headers=headers, data=f) + response.raise_for_status() + print(f"Successfully uploaded {filename}.") + return True + except requests.exceptions.RequestException as e: + print(f"Error uploading {filename}: {e}") + if e.response is not None: + print(f"Response content: {e.response.text}") + return False + + +def main(): + """Main function to download archives and upload them to the registry.""" + parser = argparse.ArgumentParser( + description="Download GitLab release archives for Eigen and upload them to the generic package registry." + ) + parser.add_argument( + "--gitlab_private_token", + type=str, + help="GitLab private API token. Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.", + ) + parser.add_argument( + "--version", + required=True, + help="Specify a single version (tag name) to process.", + ) + parser.add_argument( + "--download-dir", help=f"Directory to store temporary downloads (optional)." + ) + args = parser.parse_args() + + if not args.gitlab_private_token: + args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN") + if not args.gitlab_private_token: + print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr) + parser.print_usage() + sys.exit(1) + + # Create download directory if it doesn't exist. + cleanup_download_dir = False + if args.download_dir: + if not os.path.exists(args.download_dir): + cleanup_download_dir = True + os.makedirs(args.download_dir) + else: + args.download_dir = tempfile.mkdtemp() + cleanup_download_dir = True + + for ext in ["tar.gz", "tar.bz2", "tar", "zip"]: + archive_filename = f"eigen-{args.version}.{ext}" + archive_url = f"https://gitlab.com/libeigen/eigen/-/archive/{args.version}/{archive_filename}" + archive_filepath = os.path.join(args.download_dir, archive_filename) + + # Download the archive + print(f"Downloading {archive_url}...") + try: + response = requests.get(archive_url, stream=True) + response.raise_for_status() + with open(archive_filepath, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + print(f"Downloaded to {archive_filepath}") + except requests.exceptions.RequestException as e: + print(f"Error downloading {archive_url}: {e}. Skipping.") + continue + + # Calculate SHA256 sum + sha256_sum = calculate_sha256(archive_filepath) + print(f"SHA256 sum: {sha256_sum}") + + # Create SHA256 sum file + sha_filename = f"{archive_filename}.sha256" + sha_filepath = os.path.join(args.download_dir, sha_filename) + with open(sha_filepath, "w") as f: + f.write(f"{sha256_sum} {archive_filename}\n") + print(f"Created SHA256 file: {sha_filepath}") + + # Upload archive to generic registry + if not upload_to_generic_registry( + args.gitlab_private_token, "eigen", args.version, archive_filepath + ): + # If upload fails, clean up and move to the next release + os.remove(archive_filepath) + os.remove(sha_filepath) + continue + + # Upload SHA256 sum file to generic registry + upload_to_generic_registry( + args.gitlab_private_token, "eigen", args.version, sha_filepath + ) + + # Clean up downloaded files + print("Cleaning up local files...") + os.remove(archive_filepath) + os.remove(sha_filepath) + + # Clean up the download directory if it's empty + if cleanup_download_dir and not os.listdir(args.download_dir): + os.rmdir(args.download_dir) + + +if __name__ == "__main__": + main() diff --git a/scripts/gitlab_api_issues.py b/scripts/gitlab_api_issues.py new file mode 100644 index 000000000..1d191fa3e --- /dev/null +++ b/scripts/gitlab_api_issues.py @@ -0,0 +1,174 @@ +"""Downloads all issues from GitLab matching specified criteria.""" + +import argparse +import datetime +import json +import os +import requests +import sys + +EIGEN_PROJECT_ID = 15462818 # Taken from the gitlab project page. + + +def date(date_string: str): + """Convert a date YY-MM-DD string to a datetime object.""" + try: + return datetime.strptime(date_string, "%Y-%m-%d") + except ValueError: + msg = f"Not a valid date: '{date_string}'. Expected format is YYYY-MM-DD." + raise argparse.ArgumentTypeError(msg) + + +def _get_api_query( + gitlab_private_token: str, url: str, params: dict[str, str] | None = None +): + next_page = "1" + if not params: + params = dict() + params["per_page"] = "100" + headers = {"PRIVATE-TOKEN": gitlab_private_token} + out = [] + while next_page: + params["page"] = next_page + try: + resp = requests.head(url=url, params=params, headers=headers) + if resp.status_code != 200: + print("Request failed: ", resp, file=sys.stderr) + break + + next_next_page = resp.headers["x-next-page"] + + resp = requests.get(url=url, params=params, headers=headers) + if resp.status_code != 200: + # Try again. + continue + + out.extend(resp.json()) + # Advance at the end, in case an exception occurs above so we can retry + next_page = next_next_page + except: + # Keep same next_page + continue + return out + + +def get_issues( + gitlab_private_token: str, + author_username: str | None = None, + state: str | None = None, + created_before: datetime.datetime | None = None, + created_after: datetime.datetime | None = None, + updated_after: datetime.datetime | None = None, + updated_before: datetime.datetime | None = None, +): + """Return list of merge requests. + + Args: + gitlab_token: GitLab API token. + author_username: issue author username. + state: issue state (opened, closed). + created_after: datetime start of period. + created_before: datetime end of period. + updated_after: datetime start of period. + updated_before: datetime end of period. + + Returns: + List of merge requests. + """ + url = f"https://gitlab.com/api/v4/projects/{str(EIGEN_PROJECT_ID)}/issues" + params = dict() + if author_username: + params["author_username"] = author_username + + if state: + params["state"] = state + + if created_before: + params["created_before"] = created_before.isoformat() + + if created_after: + params["created_after"] = created_after.isoformat() + + if updated_before: + params["updated_before"] = updated_before.isoformat() + + if updated_after: + params["updated_after"] = updated_after.isoformat() + + params["order_by"] = "created_at" + params["sort"] = "asc" + issues = _get_api_query(gitlab_private_token, url, params) + for issue in issues: + if int(issue["merge_requests_count"]) > 0: + issue_iid = issue["iid"] + issue["related_merge_requests"] = _get_api_query( + gitlab_private_token, f"{url}/{issue_iid}/related_merge_requests" + ) + issue["closed_by_merge_requests"] = _get_api_query( + gitlab_private_token, f"{url}/{issue_iid}/closed_by" + ) + return issues + + +def main(_): + parser = argparse.ArgumentParser( + description="A script to download all issues from GitLab matching specified criteria." + ) + parser.add_argument( + "--gitlab_private_token", + type=str, + help="GitLab private API token. Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.", + ) + parser.add_argument("--author", type=str, help="The name of the author.") + parser.add_argument( + "--state", + type=str, + choices=["opened", "closed"], + help="The state of the issue.", + ) + parser.add_argument( + "--created_before", + type=date, + help="The created-before date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--created_after", + type=date, + help="The created-after date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--updated_before", + type=date, + help="The updated-before date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--updated_after", + type=date, + help="The updated-after date in YYYY-MM-DD format.", + ) + args = parser.parse_args() + + if not args.gitlab_private_token: + args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN") + if not args.gitlab_private_token: + print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr) + parser.print_usage() + sys.exit(1) + + # Parse the arguments from the command line + issues = get_issues( + gitlab_private_token=args.gitlab_private_token, + author_username=args.author, + state=args.state, + created_before=args.created_before, + created_after=args.created_after, + updated_before=args.updated_before, + updated_after=args.updated_after, + ) + + issue_str = json.dumps(issues, indent=2) + print(issue_str) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/gitlab_api_labeller.py b/scripts/gitlab_api_labeller.py new file mode 100644 index 000000000..f5d3530fc --- /dev/null +++ b/scripts/gitlab_api_labeller.py @@ -0,0 +1,122 @@ +"""Adds a label to a GitLab merge requests or issues.""" + +import os +import sys +import argparse +import requests + +EIGEN_PROJECT_ID = 15462818 # Taken from the gitlab project page. + + +def add_label_to_mr(private_token: str, mr_iid: int, label: str): + """ + Adds a label to a specific merge request in a GitLab project. + + Args: + private_token: The user's private GitLab API token. + mr_iid: The internal ID (IID) of the merge request. + label: The label to add. + """ + api_url = ( + f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/merge_requests/{mr_iid}" + ) + headers = {"PRIVATE-TOKEN": private_token} + # Using 'add_labels' ensures we don't overwrite existing labels. + payload = {"add_labels": label} + + try: + response = requests.put(api_url, headers=headers, json=payload) + response.raise_for_status() # Raises an HTTPError for bad responses (4xx or 5xx) + print(f"✅ Successfully added label '{label}' to Merge Request !{mr_iid}.") + except requests.exceptions.RequestException as e: + print(f"❌ Error updating Merge Request !{mr_iid}: {e}", file=sys.stderr) + if hasattr(e, "response") and e.response is not None: + print(f" Response: {e.response.text}", file=sys.stderr) + + +def add_label_to_issue(private_token: str, issue_iid: int, label: str): + """ + Adds a label to a specific issue in a GitLab project. + + Args: + private_token: The user's private GitLab API token. + issue_iid: The internal ID (IID) of the issue. + label: The label to add. + """ + api_url = ( + f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/issues/{issue_iid}" + ) + headers = {"PRIVATE-TOKEN": private_token} + payload = {"add_labels": label} + + try: + response = requests.put(api_url, headers=headers, json=payload) + response.raise_for_status() + print(f"✅ Successfully added label '{label}' to Issue #{issue_iid}.") + except requests.exceptions.RequestException as e: + print(f"❌ Error updating Issue #{issue_iid}: {e}", file=sys.stderr) + if hasattr(e, "response") and e.response is not None: + print(f" Response: {e.response.text}", file=sys.stderr) + + +def main(): + """ + Main function to parse arguments and trigger the labelling process. + """ + parser = argparse.ArgumentParser( + description="Add a label to GitLab merge requests and issues.", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument("label", help="The label to add.") + parser.add_argument( + "--mrs", + nargs="+", + type=int, + help="A space-separated list of Merge Request IIDs.", + ) + parser.add_argument( + "--issues", nargs="+", type=int, help="A space-separated list of Issue IIDs." + ) + parser.add_argument( + "--gitlab_private_token", + help="Your GitLab private access token. \n(Best practice is to use the GITLAB_PRIVATE_TOKEN environment variable instead.)", + ) + args = parser.parse_args() + + # Prefer environment variable for the token for better security. + gitlab_private_token = args.gitlab_private_token or os.environ.get( + "GITLAB_PRIVATE_TOKEN" + ) + if not gitlab_private_token: + print("Error: GitLab private token not found.", file=sys.stderr) + print( + "Please provide it using the --token argument or by setting the GITLAB_PRIVATE_TOKEN environment variable.", + file=sys.stderr, + ) + sys.exit(1) + + if not args.mrs and not args.issues: + print( + "Error: You must provide at least one merge request (--mrs) or issue (--issues) ID.", + file=sys.stderr, + ) + sys.exit(1) + + print("-" * 30) + + if args.mrs: + print(f"Processing {len(args.mrs)} merge request(s)...") + for mr_iid in args.mrs: + add_label_to_mr(gitlab_private_token, mr_iid, args.label) + + if args.issues: + print(f"\nProcessing {len(args.issues)} issue(s)...") + for issue_iid in args.issues: + add_label_to_issue(gitlab_private_token, issue_iid, args.label) + + print("-" * 30) + print("Script finished.") + + +if __name__ == "__main__": + main() diff --git a/scripts/gitlab_api_mrs.py b/scripts/gitlab_api_mrs.py new file mode 100644 index 000000000..12eb5f9b5 --- /dev/null +++ b/scripts/gitlab_api_mrs.py @@ -0,0 +1,200 @@ +"""Downloads all MRs from GitLab matching specified criteria.""" + +import argparse +import datetime +import json +import os +import requests +import sys + +EIGEN_PROJECT_ID = 15462818 # Taken from the gitlab project page. + + +def date(date_string: str): + """Convert a date YY-MM-DD string to a datetime object.""" + try: + return datetime.strptime(date_string, "%Y-%m-%d") + except ValueError: + msg = f"Not a valid date: '{date_string}'. Expected format is YYYY-MM-DD." + raise argparse.ArgumentTypeError(msg) + + +def _get_api_query( + gitlab_private_token: str, url: str, params: dict[str, str] | None = None +): + next_page = "1" + if not params: + params = dict() + params["per_page"] = "100" + headers = {"PRIVATE-TOKEN": gitlab_private_token} + out = [] + while next_page: + params["page"] = next_page + try: + resp = requests.head(url=url, params=params, headers=headers) + if resp.status_code != 200: + print("Request failed: ", resp, file=sys.stderr) + break + + next_next_page = resp.headers["x-next-page"] + + resp = requests.get(url=url, params=params, headers=headers) + if resp.status_code != 200: + # Try again. + continue + + out.extend(resp.json()) + # Advance at the end, in case an exception occurs above so we can retry + next_page = next_next_page + except: + # Keep same next_page + continue + return out + + +def get_merge_requests( + gitlab_private_token: str, + author_username: str | None = None, + state: str | None = None, + created_before: datetime.datetime | None = None, + created_after: datetime.datetime | None = None, + updated_after: datetime.datetime | None = None, + updated_before: datetime.datetime | None = None, + related_issues: bool = False, + closes_issues: bool = False, +): + """Return list of merge requests. + + Args: + gitlab_token: GitLab API token. + author_username: MR author username. + state: MR state (merged, opened, closed, locked). + created_after: datetime start of period. + created_before: datetime end of period. + updated_after: datetime start of period. + updated_before: datetime end of period. + + Returns: + List of merge requests. + """ + url = ( + "https://gitlab.com/api/v4/projects/" + + str(EIGEN_PROJECT_ID) + + "/merge_requests" + ) + params = dict() + if author_username: + params["author_username"] = author_username + + if state: + params["state"] = state + + if created_before: + params["created_before"] = created_before.isoformat() + + if created_after: + params["created_after"] = created_after.isoformat() + + if updated_before: + params["updated_before"] = updated_before.isoformat() + + if updated_after: + params["updated_after"] = updated_after.isoformat() + + params["order_by"] = "created_at" + params["sort"] = "asc" + + next_page = "1" + params["per_page"] = "100" + headers = {"PRIVATE-TOKEN": gitlab_private_token} + + mrs = _get_api_query(gitlab_private_token, url, params) + + if related_issues: + for mr in mrs: + mr["related_issues"] = _get_api_query( + gitlab_private_token, f"{url}/{mr['iid']}/related_issues" + ) + + if closes_issues: + for mr in mrs: + mr["closes_issues"] = _get_api_query( + gitlab_private_token, f"{url}/{mr['iid']}/closes_issues" + ) + + return mrs + + +def main(_): + parser = argparse.ArgumentParser( + description="A script to download all MRs from GitLab matching specified criteria." + ) + parser.add_argument( + "--gitlab_private_token", + type=str, + help="GitLab private API token. Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.", + ) + parser.add_argument("--author", type=str, help="The name of the author.") + parser.add_argument( + "--state", + type=str, + choices=["merged", "opened", "closed", "locked"], + help="The state of the MR.", + ) + parser.add_argument( + "--created_before", + type=date, + help="The created-before date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--created_after", + type=date, + help="The created-after date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--updated_before", + type=date, + help="The updated-before date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--updated_after", + type=date, + help="The updated-after date in YYYY-MM-DD format.", + ) + parser.add_argument( + "--related_issues", action="store_true", help="Query for related issues." + ) + parser.add_argument( + "--closes_issues", + action="store_true", + help="Query for issues closed by the MR.", + ) + + args = parser.parse_args() + + if not args.gitlab_private_token: + args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN") + if not args.gitlab_private_token: + print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr) + parser.print_usage() + sys.exit(1) + + # Parse the arguments from the command line + mrs = get_merge_requests( + gitlab_private_token=args.gitlab_private_token, + author_username=args.author, + state=args.state, + created_before=args.created_before, + created_after=args.created_after, + updated_before=args.updated_before, + updated_after=args.updated_after, + related_issues=args.related_issues, + closes_issues=args.closes_issues, + ) + + mr_str = json.dumps(mrs, indent=2) + print(mr_str) + + +if __name__ == "__main__": + main(sys.argv)