3.13 CI uses 3.12 for generation scripts

[ZeroIntensity]

In #142248, I noticed that my script did not work in 3.13's CI, because it used Python 3.12 instead of 3.13 or later.

The problem is in 3.13's configure script. As noted by @hugovk, 3.13's configure only checks for Python 3.13 and below, so the Python 3.14 installation used by actions/setup-python is thrown away, and an old Python 3.12.3 copy that comes with the image is used instead.

This is quite unfortunate, because for CI tools, we have to either limit ourselves to 3.12 features or create slightly different scripts for each branch (which is not ideal for backporting). In my case, I used _colorize in the job that I added, but I would have to remove that usage on the 3.13 branch, which would make it very frustrating to edit the script without conflicts.

I see two good solutions:

1. Force the actions/setup-python job to install Python 3.13 instead of 3.14.
2. Add 3.14 to the configure script.

My personal preference is the latter.

Linked PRs

• [3.13] gh-142457: Support using 3.14 and 3.15 for generation scripts #142499

[Locked-chess-official]

Use sys.version_info to control whether to import "_colorize":

import re
from pathlib import Path
import sys

_MAJOR, _MINOR = sys.version_info[:2]
if _MAJOR >= 3 and _MINOR >= 13:
    import _colorize

import textwrap

SIMPLE_FUNCTION_REGEX = re.compile(r"PyAPI_FUNC(.+) (\w+)\(")
SIMPLE_MACRO_REGEX = re.compile(r"# *define *(\w+)(\(.+\))? ")
SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)")
SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)")

CPYTHON = Path(__file__).parent.parent.parent
INCLUDE = CPYTHON / "Include"
C_API_DOCS = CPYTHON / "Doc" / "c-api"
IGNORED = (
    (CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt")
    .read_text()
    .split("\n")
)

for index, line in enumerate(IGNORED):
    if line.startswith("#"):
        IGNORED.pop(index)

MISTAKE = """
If this is a mistake and this script should not be failing, create an
issue and tag Peter (@ZeroIntensity) on it.\
"""


def found_undocumented(singular: bool) -> str:
    some = "an" if singular else "some"
    s = "" if singular else "s"
    these = "this" if singular else "these"
    them = "it" if singular else "them"
    were = "was" if singular else "were"

    return (
        textwrap.dedent(
            f"""
    Found {some} undocumented C API{s}!
    Python requires documentation on all public C API symbols, macros, and types.
    If {these} API{s} {were} not meant to be public, prefix {them} with a
    leading underscore (_PySomething_API) or move {them} to the internal C API
    (pycore_*.h files).
    In exceptional cases, certain APIs can be ignored by adding them to
    Tools/check-c-api-docs/ignored_c_api.txt
    """
        )
        + MISTAKE
    )


def found_ignored_documented(singular: bool) -> str:
    some = "a" if singular else "some"
    s = "" if singular else "s"
    them = "it" if singular else "them"
    were = "was" if singular else "were"
    they = "it" if singular else "they"

    return (
        textwrap.dedent(
            f"""
    Found {some} C API{s} listed in Tools/c-api-docs-check/ignored_c_api.txt, but
    {they} {were} found in the documentation. To fix this, remove {them} from
    ignored_c_api.txt.
    """
        )
        + MISTAKE
    )


def is_documented(name: str) -> bool:
    """
    Is a name present in the C API documentation?
    """
    for path in C_API_DOCS.iterdir():
        if path.is_dir():
            continue
        if path.suffix != ".rst":
            continue

        text = path.read_text(encoding="utf-8")
        if name in text:
            return True

    return False


def scan_file_for_docs(filename: str, text: str) -> tuple[list[str], list[str]]:
    """
    Scan a header file for  C API functions.
    """
    undocumented: list[str] = []
    documented_ignored: list[str] = []

    if _MAJOR >= 3 and _MINOR >= 13:
        colors = _colorize.get_colors()
    else:
        class colors:
            RED = "\x1b[31m"
            BOLD_RED = "\x1b[1;31m"
            BOLD_YELLOW = "\x1b[1;33m"
            RESET = "\x1b[0m"

    def check_for_name(name: str) -> None:
        documented = is_documented(name)
        if documented and (name in IGNORED):
            documented_ignored.append(name)
        elif not documented and (name not in IGNORED):
            undocumented.append(name)

    for function in SIMPLE_FUNCTION_REGEX.finditer(text):
        name = function.group(2)
        if not name.startswith("Py"):
            continue

        check_for_name(name)

    for macro in SIMPLE_MACRO_REGEX.finditer(text):
        name = macro.group(1)
        if not name.startswith("Py"):
            continue

        if "(" in name:
            name = name[: name.index("(")]

        check_for_name(name)

    for inline in SIMPLE_INLINE_REGEX.finditer(text):
        name = inline.group(2)
        if not name.startswith("Py"):
            continue

        check_for_name(name)

    for data in SIMPLE_DATA_REGEX.finditer(text):
        name = data.group(1)
        if not name.startswith("Py"):
            continue

        check_for_name(name)

    # Remove duplicates and sort alphabetically to keep the output deterministic
    undocumented = list(set(undocumented))
    undocumented.sort()

    if undocumented or documented_ignored:
        print(f"{filename} {colors.RED}BAD{colors.RESET}")
        for name in undocumented:
            print(f"{colors.BOLD_RED}UNDOCUMENTED:{colors.RESET} {name}")
        for name in documented_ignored:
            print(f"{colors.BOLD_YELLOW}DOCUMENTED BUT IGNORED:{colors.RESET} {name}")
    else:
        print(f"{filename} {colors.GREEN}OK{colors.RESET}")

    return undocumented, documented_ignored


def main() -> None:
    print("Scanning for undocumented C API functions...")
    files = [*INCLUDE.iterdir(), *(INCLUDE / "cpython").iterdir()]
    all_missing: list[str] = []
    all_found_ignored: list[str] = []

    for file in files:
        if file.is_dir():
            continue
        assert file.exists()
        text = file.read_text(encoding="utf-8")
        missing, ignored = scan_file_for_docs(str(file.relative_to(INCLUDE)), text)
        all_found_ignored += ignored
        all_missing += missing

    fail = False
    to_check = [
        (all_missing, "missing", found_undocumented(len(all_missing) == 1)),
        (
            all_found_ignored,
            "documented but ignored",
            found_ignored_documented(len(all_found_ignored) == 1),
        ),
    ]
    for name_list, what, message in to_check:
        if not name_list:
            continue

        s = "s" if len(name_list) != 1 else ""
        print(f"-- {len(name_list)} {what} C API{s} --")
        for name in name_list:
            print(f" - {name}")
        print(message)
        fail = True

    sys.exit(1 if fail else 0)


if __name__ == "__main__":
    main()

The script only use a little of the module, so it is easy to realize a smallest implement.

[ZeroIntensity]

Yeah, that's a band-aid solution, but more fundamentally, we should be able to use modern features in our CI. I would rather not clutter the code with extra branches for what is really a problem with our GHA.