Add a bunch of useful scripts for planning releases.

scripts/git_commit_mrs_and_issues.py

@@ -0,0 +1,128 @@
+"""Search for MRs and issues related to a list of commits."""
+
+import argparse
+import json
+import sys
+import subprocess
+import re
+
+
+def find_cherry_pick_source(commit_hash: str):
+    """
+    For a given commit hash, find the original commit it was cherry-picked from.
+
+    Args:
+        commit_hash: The commit hash to inspect.
+
+    Returns:
+        The full hash of the original commit if found, otherwise None.
+    """
+    try:
+        # Use 'git show' to get the full commit message for the given hash.
+        # The '-s' flag suppresses the diff output.
+        # The '--format=%B' flag prints only the raw commit body/message.
+        commit_message = subprocess.check_output(
+            ["git", "show", "-s", "--format=%B", commit_hash.strip()],
+            text=True,
+            stderr=subprocess.PIPE,
+        ).strip()
+
+        # This regex looks for the specific line Git adds during a cherry-pick.
+        # It captures the full 40-character SHA-1 hash.
+        cherry_pick_pattern = re.compile(
+            r"\(cherry picked from commit ([a-f0-9]{40})\)"
+        )
+
+        # Search the entire commit message for the pattern.
+        match = cherry_pick_pattern.search(commit_message)
+
+        if match:
+            # If a match is found, return the captured group (the original commit hash).
+            return match.group(1)
+        else:
+            return None
+
+    except subprocess.CalledProcessError as e:
+        # This error occurs if the git command fails, e.g., for an invalid hash.
+        print(
+            f"Error processing commit '{commit_hash.strip()}': {e.stderr.strip()}",
+            file=sys.stderr,
+        )
+        return None
+    except FileNotFoundError:
+        # This error occurs if the 'git' command itself isn't found.
+        print(
+            "Error: 'git' command not found. Please ensure Git is installed and in your PATH.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+
+def main():
+    """
+    Main function to read commit hashes from stdin and process them.
+    """
+    parser = argparse.ArgumentParser(
+        description="A script to download all MRs from GitLab matching specified criteria."
+    )
+    parser.add_argument(
+        "--merge_requests_file",
+        type=str,
+        required=True,
+        help="JSON file containing all the merge request information extracted via the GitLab API.",
+    )
+
+    # E.g. git log --pretty=%H 3e819d83bf52abda16bb53565f6801df40d071f1..3.4.1
+    parser.add_argument(
+        "--commits",
+        required=True,
+        help="List of commits, '-' for stdin.",
+    )
+    args = parser.parse_args()
+
+    mrs = []
+    with open(args.merge_requests_file, "r") as file:
+        mrs = json.load(file)
+        mrs_by_commit = {}
+
+    if args.commits == "-":
+        commit_hashes = sys.stdin.readlines()
+    else:
+        with open(args.commits, "r") as file:
+            commit_hashes = file.readlines()
+
+    # Arrange commits by SHA.
+    for mr in mrs:
+        for key in ["sha", "merge_commit_sha", "squash_commit_sha"]:
+            sha = mr[key]
+            if sha:
+                mrs_by_commit[sha] = mr
+
+    # Find the MRs and issues related to each commit.
+    info = {}
+    for sha in commit_hashes:
+        sha = sha.strip()
+        if not sha:
+            continue
+
+        # If a cherry-pick, extract the original hash.
+        sha = find_cherry_pick_source(sha) or sha
+        mr = mrs_by_commit.get(sha)
+
+        commit_info = {}
+        if mr:
+            commit_info["merge_request"] = mr["iid"]
+            commit_info["related_issues"] = [
+                issue["iid"] for issue in mr["related_issues"]
+            ]
+            commit_info["closes_issues"] = [
+                issue["iid"] for issue in mr["closes_issues"]
+            ]
+
+        info[sha] = commit_info
+
+    print(json.dumps(info, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/gitlab_api_deploy_package.py

@@ -0,0 +1,136 @@
+"""Helper script to download source archives and upload them to the Eigen GitLab generic package registry."""
+
+import os
+import requests
+import hashlib
+import argparse
+import sys
+import tempfile
+
+EIGEN_PROJECT_ID = 15462818  # Taken from the gitlab project page.
+
+
+def calculate_sha256(filepath: str):
+    """Calculates the SHA256 checksum of a file."""
+    sha256_hash = hashlib.sha256()
+    with open(filepath, "rb") as f:
+        # Read and update hash in chunks of 4K
+        for byte_block in iter(lambda: f.read(4096), b""):
+            sha256_hash.update(byte_block)
+    return sha256_hash.hexdigest()
+
+
+def upload_to_generic_registry(
+    gitlab_private_token: str, package_name: str, package_version: str, filepath: str
+):
+    """Uploads a file to the GitLab generic package registry."""
+    headers = {"PRIVATE-TOKEN": gitlab_private_token}
+    filename = os.path.basename(filepath)
+    upload_url = f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/packages/generic/{package_name}/{package_version}/{filename}"
+
+    print(f"Uploading {filename} to {upload_url}...")
+    try:
+        with open(filepath, "rb") as f:
+            response = requests.put(upload_url, headers=headers, data=f)
+            response.raise_for_status()
+        print(f"Successfully uploaded {filename}.")
+        return True
+    except requests.exceptions.RequestException as e:
+        print(f"Error uploading {filename}: {e}")
+        if e.response is not None:
+            print(f"Response content: {e.response.text}")
+        return False
+
+
+def main():
+    """Main function to download archives and upload them to the registry."""
+    parser = argparse.ArgumentParser(
+        description="Download GitLab release archives for Eigen and upload them to the generic package registry."
+    )
+    parser.add_argument(
+        "--gitlab_private_token",
+        type=str,
+        help="GitLab private API token.  Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.",
+    )
+    parser.add_argument(
+        "--version",
+        required=True,
+        help="Specify a single version (tag name) to process.",
+    )
+    parser.add_argument(
+        "--download-dir", help=f"Directory to store temporary downloads (optional)."
+    )
+    args = parser.parse_args()
+
+    if not args.gitlab_private_token:
+        args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN")
+        if not args.gitlab_private_token:
+            print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr)
+            parser.print_usage()
+            sys.exit(1)
+
+    # Create download directory if it doesn't exist.
+    cleanup_download_dir = False
+    if args.download_dir:
+        if not os.path.exists(args.download_dir):
+            cleanup_download_dir = True
+            os.makedirs(args.download_dir)
+    else:
+        args.download_dir = tempfile.mkdtemp()
+        cleanup_download_dir = True
+
+    for ext in ["tar.gz", "tar.bz2", "tar", "zip"]:
+        archive_filename = f"eigen-{args.version}.{ext}"
+        archive_url = f"https://gitlab.com/libeigen/eigen/-/archive/{args.version}/{archive_filename}"
+        archive_filepath = os.path.join(args.download_dir, archive_filename)
+
+        # Download the archive
+        print(f"Downloading {archive_url}...")
+        try:
+            response = requests.get(archive_url, stream=True)
+            response.raise_for_status()
+            with open(archive_filepath, "wb") as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+            print(f"Downloaded to {archive_filepath}")
+        except requests.exceptions.RequestException as e:
+            print(f"Error downloading {archive_url}: {e}. Skipping.")
+            continue
+
+        # Calculate SHA256 sum
+        sha256_sum = calculate_sha256(archive_filepath)
+        print(f"SHA256 sum: {sha256_sum}")
+
+        # Create SHA256 sum file
+        sha_filename = f"{archive_filename}.sha256"
+        sha_filepath = os.path.join(args.download_dir, sha_filename)
+        with open(sha_filepath, "w") as f:
+            f.write(f"{sha256_sum}  {archive_filename}\n")
+        print(f"Created SHA256 file: {sha_filepath}")
+
+        # Upload archive to generic registry
+        if not upload_to_generic_registry(
+            args.gitlab_private_token, "eigen", args.version, archive_filepath
+        ):
+            # If upload fails, clean up and move to the next release
+            os.remove(archive_filepath)
+            os.remove(sha_filepath)
+            continue
+
+        # Upload SHA256 sum file to generic registry
+        upload_to_generic_registry(
+            args.gitlab_private_token, "eigen", args.version, sha_filepath
+        )
+
+        # Clean up downloaded files
+        print("Cleaning up local files...")
+        os.remove(archive_filepath)
+        os.remove(sha_filepath)
+
+    # Clean up the download directory if it's empty
+    if cleanup_download_dir and not os.listdir(args.download_dir):
+        os.rmdir(args.download_dir)
+
+
+if __name__ == "__main__":
+    main()

scripts/gitlab_api_issues.py

@@ -0,0 +1,174 @@
+"""Downloads all issues from GitLab matching specified criteria."""
+
+import argparse
+import datetime
+import json
+import os
+import requests
+import sys
+
+EIGEN_PROJECT_ID = 15462818  # Taken from the gitlab project page.
+
+
+def date(date_string: str):
+    """Convert a date YY-MM-DD string to a datetime object."""
+    try:
+        return datetime.strptime(date_string, "%Y-%m-%d")
+    except ValueError:
+        msg = f"Not a valid date: '{date_string}'. Expected format is YYYY-MM-DD."
+        raise argparse.ArgumentTypeError(msg)
+
+
+def _get_api_query(
+    gitlab_private_token: str, url: str, params: dict[str, str] | None = None
+):
+    next_page = "1"
+    if not params:
+        params = dict()
+    params["per_page"] = "100"
+    headers = {"PRIVATE-TOKEN": gitlab_private_token}
+    out = []
+    while next_page:
+        params["page"] = next_page
+        try:
+            resp = requests.head(url=url, params=params, headers=headers)
+            if resp.status_code != 200:
+                print("Request failed: ", resp, file=sys.stderr)
+                break
+
+            next_next_page = resp.headers["x-next-page"]
+
+            resp = requests.get(url=url, params=params, headers=headers)
+            if resp.status_code != 200:
+                # Try again.
+                continue
+
+            out.extend(resp.json())
+            # Advance at the end, in case an exception occurs above so we can retry
+            next_page = next_next_page
+        except:
+            # Keep same next_page
+            continue
+    return out
+
+
+def get_issues(
+    gitlab_private_token: str,
+    author_username: str | None = None,
+    state: str | None = None,
+    created_before: datetime.datetime | None = None,
+    created_after: datetime.datetime | None = None,
+    updated_after: datetime.datetime | None = None,
+    updated_before: datetime.datetime | None = None,
+):
+    """Return list of merge requests.
+
+    Args:
+      gitlab_token: GitLab API token.
+      author_username: issue author username.
+      state: issue state (opened, closed).
+      created_after: datetime start of period.
+      created_before: datetime end of period.
+      updated_after: datetime start of period.
+      updated_before: datetime end of period.
+
+    Returns:
+      List of merge requests.
+    """
+    url = f"https://gitlab.com/api/v4/projects/{str(EIGEN_PROJECT_ID)}/issues"
+    params = dict()
+    if author_username:
+        params["author_username"] = author_username
+
+    if state:
+        params["state"] = state
+
+    if created_before:
+        params["created_before"] = created_before.isoformat()
+
+    if created_after:
+        params["created_after"] = created_after.isoformat()
+
+    if updated_before:
+        params["updated_before"] = updated_before.isoformat()
+
+    if updated_after:
+        params["updated_after"] = updated_after.isoformat()
+
+    params["order_by"] = "created_at"
+    params["sort"] = "asc"
+    issues = _get_api_query(gitlab_private_token, url, params)
+    for issue in issues:
+        if int(issue["merge_requests_count"]) > 0:
+            issue_iid = issue["iid"]
+            issue["related_merge_requests"] = _get_api_query(
+                gitlab_private_token, f"{url}/{issue_iid}/related_merge_requests"
+            )
+            issue["closed_by_merge_requests"] = _get_api_query(
+                gitlab_private_token, f"{url}/{issue_iid}/closed_by"
+            )
+    return issues
+
+
+def main(_):
+    parser = argparse.ArgumentParser(
+        description="A script to download all issues from GitLab matching specified criteria."
+    )
+    parser.add_argument(
+        "--gitlab_private_token",
+        type=str,
+        help="GitLab private API token.  Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.",
+    )
+    parser.add_argument("--author", type=str, help="The name of the author.")
+    parser.add_argument(
+        "--state",
+        type=str,
+        choices=["opened", "closed"],
+        help="The state of the issue.",
+    )
+    parser.add_argument(
+        "--created_before",
+        type=date,
+        help="The created-before date in YYYY-MM-DD format.",
+    )
+    parser.add_argument(
+        "--created_after",
+        type=date,
+        help="The created-after date in YYYY-MM-DD format.",
+    )
+    parser.add_argument(
+        "--updated_before",
+        type=date,
+        help="The updated-before date in YYYY-MM-DD format.",
+    )
+    parser.add_argument(
+        "--updated_after",
+        type=date,
+        help="The updated-after date in YYYY-MM-DD format.",
+    )
+    args = parser.parse_args()
+
+    if not args.gitlab_private_token:
+        args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN")
+        if not args.gitlab_private_token:
+            print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr)
+            parser.print_usage()
+            sys.exit(1)
+
+    # Parse the arguments from the command line
+    issues = get_issues(
+        gitlab_private_token=args.gitlab_private_token,
+        author_username=args.author,
+        state=args.state,
+        created_before=args.created_before,
+        created_after=args.created_after,
+        updated_before=args.updated_before,
+        updated_after=args.updated_after,
+    )
+
+    issue_str = json.dumps(issues, indent=2)
+    print(issue_str)
+
+
+if __name__ == "__main__":
+    main(sys.argv)

scripts/gitlab_api_labeller.py

@@ -0,0 +1,122 @@
+"""Adds a label to a GitLab merge requests or issues."""
+
+import os
+import sys
+import argparse
+import requests
+
+EIGEN_PROJECT_ID = 15462818  # Taken from the gitlab project page.
+
+
+def add_label_to_mr(private_token: str, mr_iid: int, label: str):
+    """
+    Adds a label to a specific merge request in a GitLab project.
+
+    Args:
+        private_token: The user's private GitLab API token.
+        mr_iid: The internal ID (IID) of the merge request.
+        label: The label to add.
+    """
+    api_url = (
+        f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/merge_requests/{mr_iid}"
+    )
+    headers = {"PRIVATE-TOKEN": private_token}
+    # Using 'add_labels' ensures we don't overwrite existing labels.
+    payload = {"add_labels": label}
+
+    try:
+        response = requests.put(api_url, headers=headers, json=payload)
+        response.raise_for_status()  # Raises an HTTPError for bad responses (4xx or 5xx)
+        print(f"✅ Successfully added label '{label}' to Merge Request !{mr_iid}.")
+    except requests.exceptions.RequestException as e:
+        print(f"❌ Error updating Merge Request !{mr_iid}: {e}", file=sys.stderr)
+        if hasattr(e, "response") and e.response is not None:
+            print(f"   Response: {e.response.text}", file=sys.stderr)
+
+
+def add_label_to_issue(private_token: str, issue_iid: int, label: str):
+    """
+    Adds a label to a specific issue in a GitLab project.
+
+    Args:
+        private_token: The user's private GitLab API token.
+        issue_iid: The internal ID (IID) of the issue.
+        label: The label to add.
+    """
+    api_url = (
+        f"https://gitlab.com/api/v4/projects/{EIGEN_PROJECT_ID}/issues/{issue_iid}"
+    )
+    headers = {"PRIVATE-TOKEN": private_token}
+    payload = {"add_labels": label}
+
+    try:
+        response = requests.put(api_url, headers=headers, json=payload)
+        response.raise_for_status()
+        print(f"✅ Successfully added label '{label}' to Issue #{issue_iid}.")
+    except requests.exceptions.RequestException as e:
+        print(f"❌ Error updating Issue #{issue_iid}: {e}", file=sys.stderr)
+        if hasattr(e, "response") and e.response is not None:
+            print(f"   Response: {e.response.text}", file=sys.stderr)
+
+
+def main():
+    """
+    Main function to parse arguments and trigger the labelling process.
+    """
+    parser = argparse.ArgumentParser(
+        description="Add a label to GitLab merge requests and issues.",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    parser.add_argument("label", help="The label to add.")
+    parser.add_argument(
+        "--mrs",
+        nargs="+",
+        type=int,
+        help="A space-separated list of Merge Request IIDs.",
+    )
+    parser.add_argument(
+        "--issues", nargs="+", type=int, help="A space-separated list of Issue IIDs."
+    )
+    parser.add_argument(
+        "--gitlab_private_token",
+        help="Your GitLab private access token. \n(Best practice is to use the GITLAB_PRIVATE_TOKEN environment variable instead.)",
+    )
+    args = parser.parse_args()
+
+    # Prefer environment variable for the token for better security.
+    gitlab_private_token = args.gitlab_private_token or os.environ.get(
+        "GITLAB_PRIVATE_TOKEN"
+    )
+    if not gitlab_private_token:
+        print("Error: GitLab private token not found.", file=sys.stderr)
+        print(
+            "Please provide it using the --token argument or by setting the GITLAB_PRIVATE_TOKEN environment variable.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    if not args.mrs and not args.issues:
+        print(
+            "Error: You must provide at least one merge request (--mrs) or issue (--issues) ID.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    print("-" * 30)
+
+    if args.mrs:
+        print(f"Processing {len(args.mrs)} merge request(s)...")
+        for mr_iid in args.mrs:
+            add_label_to_mr(gitlab_private_token, mr_iid, args.label)
+
+    if args.issues:
+        print(f"\nProcessing {len(args.issues)} issue(s)...")
+        for issue_iid in args.issues:
+            add_label_to_issue(gitlab_private_token, issue_iid, args.label)
+
+    print("-" * 30)
+    print("Script finished.")
+
+
+if __name__ == "__main__":
+    main()

scripts/gitlab_api_mrs.py

@@ -0,0 +1,200 @@
+"""Downloads all MRs from GitLab matching specified criteria."""
+
+import argparse
+import datetime
+import json
+import os
+import requests
+import sys
+
+EIGEN_PROJECT_ID = 15462818  # Taken from the gitlab project page.
+
+
+def date(date_string: str):
+    """Convert a date YY-MM-DD string to a datetime object."""
+    try:
+        return datetime.strptime(date_string, "%Y-%m-%d")
+    except ValueError:
+        msg = f"Not a valid date: '{date_string}'. Expected format is YYYY-MM-DD."
+        raise argparse.ArgumentTypeError(msg)
+
+
+def _get_api_query(
+    gitlab_private_token: str, url: str, params: dict[str, str] | None = None
+):
+    next_page = "1"
+    if not params:
+        params = dict()
+    params["per_page"] = "100"
+    headers = {"PRIVATE-TOKEN": gitlab_private_token}
+    out = []
+    while next_page:
+        params["page"] = next_page
+        try:
+            resp = requests.head(url=url, params=params, headers=headers)
+            if resp.status_code != 200:
+                print("Request failed: ", resp, file=sys.stderr)
+                break
+
+            next_next_page = resp.headers["x-next-page"]
+
+            resp = requests.get(url=url, params=params, headers=headers)
+            if resp.status_code != 200:
+                # Try again.
+                continue
+
+            out.extend(resp.json())
+            # Advance at the end, in case an exception occurs above so we can retry
+            next_page = next_next_page
+        except:
+            # Keep same next_page
+            continue
+    return out
+
+
+def get_merge_requests(
+    gitlab_private_token: str,
+    author_username: str | None = None,
+    state: str | None = None,
+    created_before: datetime.datetime | None = None,
+    created_after: datetime.datetime | None = None,
+    updated_after: datetime.datetime | None = None,
+    updated_before: datetime.datetime | None = None,
+    related_issues: bool = False,
+    closes_issues: bool = False,
+):
+    """Return list of merge requests.
+
+    Args:
+      gitlab_token: GitLab API token.
+      author_username: MR author username.
+      state: MR state (merged, opened, closed, locked).
+      created_after: datetime start of period.
+      created_before: datetime end of period.
+      updated_after: datetime start of period.
+      updated_before: datetime end of period.
+
+    Returns:
+      List of merge requests.
+    """
+    url = (
+        "https://gitlab.com/api/v4/projects/"
+        + str(EIGEN_PROJECT_ID)
+        + "/merge_requests"
+    )
+    params = dict()
+    if author_username:
+        params["author_username"] = author_username
+
+    if state:
+        params["state"] = state
+
+    if created_before:
+        params["created_before"] = created_before.isoformat()
+
+    if created_after:
+        params["created_after"] = created_after.isoformat()
+
+    if updated_before:
+        params["updated_before"] = updated_before.isoformat()
+
+    if updated_after:
+        params["updated_after"] = updated_after.isoformat()
+
+    params["order_by"] = "created_at"
+    params["sort"] = "asc"
+
+    next_page = "1"
+    params["per_page"] = "100"
+    headers = {"PRIVATE-TOKEN": gitlab_private_token}
+
+    mrs = _get_api_query(gitlab_private_token, url, params)
+
+    if related_issues:
+        for mr in mrs:
+            mr["related_issues"] = _get_api_query(
+                gitlab_private_token, f"{url}/{mr['iid']}/related_issues"
+            )
+
+    if closes_issues:
+        for mr in mrs:
+            mr["closes_issues"] = _get_api_query(
+                gitlab_private_token, f"{url}/{mr['iid']}/closes_issues"
+            )
+
+    return mrs
+
+
+def main(_):
+    parser = argparse.ArgumentParser(
+        description="A script to download all MRs from GitLab matching specified criteria."
+    )
+    parser.add_argument(
+        "--gitlab_private_token",
+        type=str,
+        help="GitLab private API token.  Defaults to the GITLAB_PRIVATE_TOKEN environment variable if set.",
+    )
+    parser.add_argument("--author", type=str, help="The name of the author.")
+    parser.add_argument(
+        "--state",
+        type=str,
+        choices=["merged", "opened", "closed", "locked"],
+        help="The state of the MR.",
+    )
+    parser.add_argument(
+        "--created_before",
+        type=date,
+        help="The created-before date in YYYY-MM-DD format.",
+    )
+    parser.add_argument(
+        "--created_after",
+        type=date,
+        help="The created-after date in YYYY-MM-DD format.",
+    )
+    parser.add_argument(
+        "--updated_before",
+        type=date,
+        help="The updated-before date in YYYY-MM-DD format.",
+    )
+    parser.add_argument(
+        "--updated_after",
+        type=date,
+        help="The updated-after date in YYYY-MM-DD format.",
+    )
+    parser.add_argument(
+        "--related_issues", action="store_true", help="Query for related issues."
+    )
+    parser.add_argument(
+        "--closes_issues",
+        action="store_true",
+        help="Query for issues closed by the MR.",
+    )
+
+    args = parser.parse_args()
+
+    if not args.gitlab_private_token:
+        args.gitlab_private_token = os.getenv("GITLAB_PRIVATE_TOKEN")
+        if not args.gitlab_private_token:
+            print("Could not determine GITLAB_PRIVATE_TOKEN.", file=sys.stderr)
+            parser.print_usage()
+            sys.exit(1)
+
+    # Parse the arguments from the command line
+    mrs = get_merge_requests(
+        gitlab_private_token=args.gitlab_private_token,
+        author_username=args.author,
+        state=args.state,
+        created_before=args.created_before,
+        created_after=args.created_after,
+        updated_before=args.updated_before,
+        updated_after=args.updated_after,
+        related_issues=args.related_issues,
+        closes_issues=args.closes_issues,
+    )
+
+    mr_str = json.dumps(mrs, indent=2)
+    print(mr_str)
+
+
+if __name__ == "__main__":
+    main(sys.argv)