diff --git a/.github/scripts/select_openapi_prs.py b/.github/scripts/select_openapi_prs.py new file mode 100644 index 0000000000..b6d3d761f9 --- /dev/null +++ b/.github/scripts/select_openapi_prs.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Select the OpenAPI description update PRs to merge, and the superseded ones. + +Reads the pages produced by + + gh api --paginate --slurp "repos/{repo}/pulls?state=open&per_page=100" + +on stdin and writes `key=value` lines suitable for `$GITHUB_OUTPUT`. + +Every open pull request is read, following pagination to the end, because a +capped listing would silently drop the oldest matches. Those are exactly the +superseded pull requests that must be closed: one left open becomes the +newest open pull request for its title once the merged one closes, and +merging it would put an earlier description back on the default branch. + +Two properties matter for safety and are the reason this is a script rather +than an inline `jq` filter: + +* Only pull requests whose title is *exactly* one of the two recognised + description-update titles are considered. Any other pull request by the + same author is neither merged nor closed. +* A title only contributes superseded pull requests when its own newest + pull request was selected, so nothing is closed unless a validated + replacement for that exact title is being merged. +* Superseded numbers are emitted per title rather than in one list, so the + workflow can confirm a pull request still carries the title it was + selected under before closing it. +* The head commit SHA of each selected pull request is emitted alongside its + number, so every later step can pin to the exact commit that was inspected + instead of resolving a branch name again. +* The author is matched here rather than by a listing flag, because the + endpoint that pages over every open pull request cannot filter by author. + Keeping the author and title rules together means one tested place decides + what is eligible. +""" + +import argparse +import json +import sys + +TITLE_30 = 'Update OpenAPI 3.0 Descriptions' +TITLE_31 = 'Update OpenAPI 3.1 Descriptions' +TITLES = (TITLE_30, TITLE_31) +# Output suffix -> exact title. The workflow reads a title through `--title` +# so the two files never drift apart. +GROUPS = (('30', TITLE_30), ('31', TITLE_31)) + + +def normalise(payload, author): + """Flatten paginated pages and map API fields to the names used below. + + `--slurp` yields one list per page, so the payload is a list of lists. + A single flat list is accepted too, which keeps the parsing rules the + same whichever way the input was produced. + """ + flat = [] + for entry in payload: + if isinstance(entry, list): + flat.extend(entry) + else: + flat.append(entry) + + normalised = [] + for pr in flat: + if not isinstance(pr, dict): + continue + # An entry whose author cannot be read is skipped rather than + # assumed to match: it must never become a merge candidate, and it + # must never be closed as superseded. + user = pr.get('user') + login = user.get('login') if isinstance(user, dict) else None + if login != author: + continue + head = pr.get('head') + normalised.append( + { + 'number': pr.get('number'), + 'title': pr.get('title'), + 'headRefOid': head.get('sha') if isinstance(head, dict) else None, + 'createdAt': pr.get('created_at'), + } + ) + + return normalised + + +def _sort_key(pr): + # `createdAt` is RFC 3339 in UTC, so lexical order is chronological. + # The number breaks ties deterministically. + return (pr.get('createdAt') or '', pr.get('number') or 0) + + +def select(pull_requests): + """Return the pinnable newest PR per recognised title, and the superseded. + + A title only contributes superseded pull requests when its own newest + pull request was actually selected. Nothing is ever closed on the basis + of a replacement that was not validated. The superseded numbers are keyed + by title so the caller can re-check that association before closing. + """ + recognised = [ + pr + for pr in pull_requests + if isinstance(pr, dict) and pr.get('title') in TITLES and pr.get('number') + ] + + selected = {} + superseded = {title: [] for title in TITLES} + for title in TITLES: + candidates = [pr for pr in recognised if pr['title'] == title] + if not candidates: + continue + newest = max(candidates, key=_sort_key) + # Without a head SHA the newest pull request cannot be pinned to an + # exact commit, so it is not merged from a mutable branch instead -- + # and none of its older siblings are closed. + if not newest.get('headRefOid'): + continue + selected[title] = newest + superseded[title] = sorted( + pr['number'] for pr in candidates if pr['number'] != newest['number'] + ) + + return selected, superseded + + +def outputs(pull_requests): + selected, superseded = select(pull_requests) + + lines = [f'found={"true" if selected else "false"}'] + for suffix, title in GROUPS: + pr = selected.get(title) + lines.append(f'pr_{suffix}={pr["number"] if pr else ""}') + lines.append(f'sha_{suffix}={pr["headRefOid"] if pr else ""}') + lines.append( + f'superseded_{suffix}=' + ' '.join(str(n) for n in superseded[title]) + ) + + return lines + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + '--author', + help='only consider pull requests opened by this login', + ) + parser.add_argument( + '--title', + choices=[suffix for suffix, _ in GROUPS], + help='print the exact recognised title for one output group and exit', + ) + args = parser.parse_args(argv) + + if args.title: + print(dict(GROUPS)[args.title]) + return 0 + + if not args.author: + print('--author is required when selecting', file=sys.stderr) + return 2 + + payload = json.load(sys.stdin) + if not isinstance(payload, list): + print('expected a JSON array of pull requests', file=sys.stderr) + return 2 + + for line in outputs(normalise(payload, args.author)): + print(line) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/test_select_openapi_prs.py b/.github/scripts/test_select_openapi_prs.py new file mode 100644 index 0000000000..45617fc89c --- /dev/null +++ b/.github/scripts/test_select_openapi_prs.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Tests for select_openapi_prs. + +Run with: python3 -m unittest discover -s .github/scripts +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import select_openapi_prs as sel # noqa: E402 + + +def pr(number, title, sha=None, created='2026-01-01T00:00:00Z'): + return { + 'number': number, + 'title': title, + 'headRefOid': sha if sha is not None else f'{number:040x}', + 'createdAt': created, + } + + +def as_dict(lines): + return dict(line.split('=', 1) for line in lines) + + +class SelectionTest(unittest.TestCase): + def test_selects_newest_per_title(self): + prs = [ + pr(1, sel.TITLE_30, created='2026-01-01T00:00:00Z'), + pr(2, sel.TITLE_30, created='2026-01-03T00:00:00Z'), + pr(3, sel.TITLE_31, created='2026-01-02T00:00:00Z'), + ] + out = as_dict(sel.outputs(prs)) + self.assertEqual(out['found'], 'true') + self.assertEqual(out['pr_30'], '2') + self.assertEqual(out['pr_31'], '3') + self.assertEqual(out['superseded_30'], '1') + self.assertEqual(out['superseded_31'], '') + + def test_ties_broken_by_number(self): + same = '2026-01-01T00:00:00Z' + out = as_dict(sel.outputs([ + pr(10, sel.TITLE_30, created=same), + pr(11, sel.TITLE_30, created=same), + ])) + self.assertEqual(out['pr_30'], '11') + self.assertEqual(out['superseded_30'], '10') + + def test_unrelated_bot_pull_requests_are_never_superseded(self): + prs = [ + pr(1, sel.TITLE_30, created='2026-01-01T00:00:00Z'), + pr(2, sel.TITLE_30, created='2026-01-02T00:00:00Z'), + pr(3, 'Bump some dependency'), + pr(4, 'Update OpenAPI 3.0 Descriptions '), + pr(5, 'update openapi 3.0 descriptions'), + pr(6, 'Update OpenAPI 3.2 Descriptions'), + pr(7, 'Revert "Update OpenAPI 3.0 Descriptions"'), + ] + out = as_dict(sel.outputs(prs)) + self.assertEqual(out['pr_30'], '2') + self.assertEqual(out['superseded_30'], '1') + self.assertEqual(out['superseded_31'], '') + + def test_only_unrelated_pull_requests_means_nothing_to_do(self): + out = as_dict(sel.outputs([pr(1, 'Bump some dependency')])) + self.assertEqual(out['found'], 'false') + self.assertEqual(out['pr_30'], '') + self.assertEqual(out['pr_31'], '') + self.assertEqual(out['superseded_30'], '') + self.assertEqual(out['superseded_31'], '') + + def test_empty_input(self): + out = as_dict(sel.outputs([])) + self.assertEqual(out['found'], 'false') + self.assertEqual(out['superseded_30'], '') + self.assertEqual(out['superseded_31'], '') + + def test_one_title_only(self): + out = as_dict(sel.outputs([pr(9, sel.TITLE_31)])) + self.assertEqual(out['found'], 'true') + self.assertEqual(out['pr_30'], '') + self.assertEqual(out['sha_30'], '') + self.assertEqual(out['pr_31'], '9') + self.assertEqual(out['superseded_31'], '') + + +class ShaPinningTest(unittest.TestCase): + def test_head_sha_is_emitted_for_each_selection(self): + out = as_dict(sel.outputs([ + pr(1, sel.TITLE_30, sha='a' * 40), + pr(2, sel.TITLE_31, sha='b' * 40), + ])) + self.assertEqual(out['sha_30'], 'a' * 40) + self.assertEqual(out['sha_31'], 'b' * 40) + + def test_sha_tracks_the_newest_pull_request_not_the_first(self): + out = as_dict(sel.outputs([ + pr(1, sel.TITLE_30, sha='a' * 40, created='2026-01-01T00:00:00Z'), + pr(2, sel.TITLE_30, sha='c' * 40, created='2026-02-01T00:00:00Z'), + ])) + self.assertEqual(out['pr_30'], '2') + self.assertEqual(out['sha_30'], 'c' * 40) + + def test_pull_request_without_head_sha_blocks_closing_its_siblings(self): + prs = [ + pr(1, sel.TITLE_30, created='2026-01-01T00:00:00Z'), + pr(2, sel.TITLE_30, sha='', created='2026-01-02T00:00:00Z'), + ] + out = as_dict(sel.outputs(prs)) + self.assertEqual(out['found'], 'false') + self.assertEqual(out['pr_30'], '') + self.assertEqual(out['sha_30'], '') + # #1 is older, but its replacement was never validated, so it must + # not be closed. + self.assertEqual(out['superseded_30'], '') + + def test_unpinnable_title_does_not_block_the_other_title(self): + prs = [ + pr(1, sel.TITLE_30, created='2026-01-01T00:00:00Z'), + pr(2, sel.TITLE_30, sha='', created='2026-01-02T00:00:00Z'), + pr(3, sel.TITLE_31, created='2026-01-01T00:00:00Z'), + pr(4, sel.TITLE_31, created='2026-01-02T00:00:00Z'), + ] + out = as_dict(sel.outputs(prs)) + self.assertEqual(out['found'], 'true') + self.assertEqual(out['pr_30'], '') + self.assertEqual(out['pr_31'], '4') + # Only the 3.1 sibling is superseded; the 3.0 pair is left alone. + self.assertEqual(out['superseded_30'], '') + self.assertEqual(out['superseded_31'], '3') + + def test_superseded_numbers_stay_with_their_title(self): + prs = [ + pr(1, sel.TITLE_30, created='2026-01-01T00:00:00Z'), + pr(2, sel.TITLE_30, created='2026-01-04T00:00:00Z'), + pr(3, sel.TITLE_31, created='2026-01-02T00:00:00Z'), + pr(4, sel.TITLE_31, created='2026-01-03T00:00:00Z'), + ] + _, superseded = sel.select(prs) + # The association is what lets the workflow confirm a pull request + # still carries the title it was superseded under before closing it. + self.assertEqual(superseded[sel.TITLE_30], [1]) + self.assertEqual(superseded[sel.TITLE_31], [3]) + + def test_malformed_entries_are_ignored(self): + out = as_dict(sel.outputs([ + 'not a dict', + {'title': sel.TITLE_30}, + pr(5, sel.TITLE_30), + ])) + self.assertEqual(out['pr_30'], '5') + self.assertEqual(out['superseded_30'], '') + + +def api_pr(number, title, sha='sha', created='2026-01-01T00:00:00Z', login='bot'): + """One entry shaped like the REST pulls endpoint returns it.""" + return { + 'number': number, + 'title': title, + 'head': {'sha': sha}, + 'created_at': created, + 'user': {'login': login}, + } + + +class NormaliseTest(unittest.TestCase): + """Reading every page matters: a dropped page leaves superseded PRs open.""" + + def test_pages_are_flattened_and_fields_mapped(self): + pages = [ + [api_pr(1, sel.TITLE_30, sha='a', created='2026-01-01T00:00:00Z')], + [api_pr(2, sel.TITLE_30, sha='b', created='2026-02-01T00:00:00Z')], + ] + self.assertEqual( + sel.normalise(pages, 'bot'), + [ + { + 'number': 1, + 'title': sel.TITLE_30, + 'headRefOid': 'a', + 'createdAt': '2026-01-01T00:00:00Z', + }, + { + 'number': 2, + 'title': sel.TITLE_30, + 'headRefOid': 'b', + 'createdAt': '2026-02-01T00:00:00Z', + }, + ], + ) + + def test_a_flat_list_is_accepted_too(self): + self.assertEqual(len(sel.normalise([api_pr(1, sel.TITLE_30)], 'bot')), 1) + + def test_another_author_is_never_eligible(self): + pages = [[api_pr(1, sel.TITLE_30, login='someone-else')]] + self.assertEqual(sel.normalise(pages, 'bot'), []) + + def test_an_unreadable_author_is_skipped_rather_than_assumed(self): + pages = [[{'number': 1, 'title': sel.TITLE_30, 'head': {'sha': 'a'}}]] + self.assertEqual(sel.normalise(pages, 'bot'), []) + + def test_a_malformed_entry_does_not_abort_the_page(self): + pages = [['not a dict', api_pr(2, sel.TITLE_30), None]] + self.assertEqual([pr['number'] for pr in sel.normalise(pages, 'bot')], [2]) + + def test_a_missing_head_survives_normalisation_and_blocks_pinning(self): + pages = [[{'number': 1, 'title': sel.TITLE_30, 'user': {'login': 'bot'}}]] + normalised = sel.normalise(pages, 'bot') + self.assertIsNone(normalised[0]['headRefOid']) + self.assertEqual(sel.outputs(normalised)[0], 'found=false') + + def test_an_older_pull_request_on_a_later_page_is_still_superseded(self): + # The regression this guards: the oldest matches arrive last, and + # dropping them would leave a stale pull request open to be selected + # by a later run once the merged one closes. + pages = [ + [api_pr(9, sel.TITLE_30, sha='new', created='2026-05-01T00:00:00Z')], + [api_pr(1, sel.TITLE_30, sha='old', created='2026-01-01T00:00:00Z')], + ] + out = dict(line.split('=', 1) for line in sel.outputs(sel.normalise(pages, 'bot'))) + self.assertEqual(out['pr_30'], '9') + self.assertEqual(out['sha_30'], 'new') + self.assertEqual(out['superseded_30'], '1') + + +class CliTest(unittest.TestCase): + def _run(self, argv): + import io + + captured, sys.stdout = sys.stdout, io.StringIO() + try: + code = sel.main(argv) + printed = sys.stdout.getvalue().splitlines() + finally: + sys.stdout = captured + return code, printed + + def test_title_flag_prints_the_exact_title_for_a_group(self): + self.assertEqual(self._run(['--title', '30']), (0, [sel.TITLE_30])) + self.assertEqual(self._run(['--title', '31']), (0, [sel.TITLE_31])) + + def test_selecting_without_an_author_is_refused(self): + import io + + captured, sys.stderr = sys.stderr, io.StringIO() + try: + self.assertEqual(sel.main([]), 2) + finally: + sys.stderr = captured + + def test_title_flag_rejects_an_unknown_group(self): + import io + + captured, sys.stderr = sys.stderr, io.StringIO() + try: + with self.assertRaises(SystemExit): + sel.main(['--title', '32']) + finally: + sys.stderr = captured + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/auto-merge-openapi-updates.yml b/.github/workflows/auto-merge-openapi-updates.yml new file mode 100644 index 0000000000..b3f3649b2f --- /dev/null +++ b/.github/workflows/auto-merge-openapi-updates.yml @@ -0,0 +1,687 @@ +name: Auto-merge OpenAPI description updates + +# Merges the newest open `github-openapi-bot` "Update OpenAPI 3.x Descriptions" +# pull requests and closes the older superseded ones. +# +# Why a workflow rather than native auto-merge: these pull requests carry 100K+ +# line diffs across 64+ files, and `GET /pulls/{n}/files` returns 422 on them, +# so the list of changed files is read from a blobless clone instead. The merge +# itself goes through `PUT /pulls/{n}/merge`, because the default branch +# requires a pull request and so refuses a direct push; that call can time out +# at the gateway on diffs this size, which the merge step handles. +# +# Safety properties: +# * Only pull requests titled exactly "Update OpenAPI 3.0 Descriptions" or +# "Update OpenAPI 3.1 Descriptions" are ever merged or closed, and a +# superseded pull request is re-checked against the exact title it was +# superseded under, immediately before it is closed. +# * Every step pins to the head commit SHA captured during selection. Branch +# names are never re-resolved, and the merge call itself requires that SHA, +# so a push that lands mid-run cannot slip an unvalidated commit into a +# merge. +# * Lint must be green, the candidate must touch nothing outside the +# description directories, and a change summary is posted to the pull +# request before merge. +# * Compatibility review is not repeated here. These descriptions are +# generated and reviewed for compatibility before publication, so a +# re-check against the published files would be a weaker duplicate of a +# gate that has already run against the authoritative source. +# * Every decision is made by deterministic code, so the same inputs always +# produce the same merge/no-merge answer and it can be tested. +# * Merges are held during a GHES release-candidate window. See the "Check +# for an active merge freeze" step. +# +# Scripts are run from a checkout of this repository's default branch, never +# from the candidate pull request. + +on: + schedule: + - cron: '17 */2 * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Analyze and report, but do not merge or close anything' + type: boolean + default: false + +permissions: + contents: write + pull-requests: write + # Reading check runs for the candidate commits. + checks: read + # Reading labels and issues for the merge-freeze signal. + issues: read + +concurrency: + group: auto-merge-openapi-updates + cancel-in-progress: false + +env: + BOT_LOGIN: github-openapi-bot + # Status checks that must be green before merging. CodeQL is deliberately + # excluded: default setup only scans the `actions` language, it is not a + # required check on the default branch, and it routinely reports `timed_out` + # on these pull requests. + REQUIRED_CHECKS: 'Lint OpenAPI 3.0 releases,Lint OpenAPI 3.1 releases' + +jobs: + auto-merge: + name: Auto-merge OpenAPI updates + runs-on: ubuntu-latest + outputs: + status: ${{ steps.merge.outputs.status }} + detail: ${{ steps.merge.outputs.detail }} + steps: + - name: Checkout workflow scripts + uses: actions/checkout@v4 + with: + # Pinned to the default branch on purpose. A `workflow_dispatch` run + # can be started from any ref, and without this the write-enabled job + # would run the selector from that ref instead. + ref: ${{ github.event.repository.default_branch }} + path: tools + fetch-depth: 1 + sparse-checkout: | + .github/scripts + sparse-checkout-cone-mode: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Select candidate pull requests + id: select + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + + # Every open pull request by the bot, following pagination rather + # than stopping at a fixed count. A cap that silently drops the + # oldest matches would leave superseded pull requests open, and a + # later run would then treat one of them as the newest open + # candidate and put an earlier description back on the default + # branch. During a long freeze these accumulate well past one page. + gh api --paginate --slurp \ + "repos/$GH_REPO/pulls?state=open&per_page=100" \ + | python3 tools/.github/scripts/select_openapi_prs.py \ + --author "$BOT_LOGIN" >selection.txt + + cat selection.txt + cat selection.txt >>"$GITHUB_OUTPUT" + + - name: Verify required checks are green + id: checks + if: steps.select.outputs.found == 'true' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + SHA_30: ${{ steps.select.outputs.sha_30 }} + SHA_31: ${{ steps.select.outputs.sha_31 }} + PR_30: ${{ steps.select.outputs.pr_30 }} + PR_31: ${{ steps.select.outputs.pr_31 }} + run: | + set -euo pipefail + + blocked='' + for pair in "$PR_30:$SHA_30" "$PR_31:$SHA_31"; do + pr="${pair%%:*}"; sha="${pair#*:}" + [ -n "$pr" ] && [ -n "$sha" ] || continue + + # Checks are read for the exact commit selected above, which is + # also the commit that is scanned and merged. + runs=$(gh api "repos/$GH_REPO/commits/$sha/check-runs" --paginate \ + -q '.check_runs[] | "\(.name)\t\(.status)\t\(.conclusion)"') + + IFS=',' read -ra required <<<"$REQUIRED_CHECKS" + for name in "${required[@]}"; do + matches=$(awk -F'\t' -v n="$name" '$1 == n' <<<"$runs") + if [ -z "$matches" ]; then + blocked+="PR #$pr: required check '$name' has not reported. " + continue + fi + if grep -qv $'\tcompleted\tsuccess$' <<<"$matches"; then + blocked+="PR #$pr: check '$name' is not passing. " + fi + done + done + + if [ -n "$blocked" ]; then + echo "status=blocked" >>"$GITHUB_OUTPUT" + echo "detail=$blocked" >>"$GITHUB_OUTPUT" + echo "::notice::$blocked" + else + echo "status=green" >>"$GITHUB_OUTPUT" + fi + + - name: Check for an active merge freeze + id: freeze + if: steps.checks.outputs.status == 'green' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + SHA_30: ${{ steps.select.outputs.sha_30 }} + SHA_31: ${{ steps.select.outputs.sha_31 }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + + # Around a GHES release candidate, hold description merges until the + # coordinated release content is ready. A new GHES version appearing + # in the candidate before it exists on the default branch is the + # signal that this window may be active. + + reason='' + + # --- Signal 1: an explicit, human-controlled hold ---------------- + # A label is the escape hatch that does not depend on inference, and + # lets a human stop merges for any reason at all. + # Output is captured before matching: under `pipefail`, an early + # exiting `grep -q` could otherwise make the producer look failed + # and silently drop this signal. + labels=$(gh label list --search 'merge-freeze' --json name -q '.[].name') + if grep -qxF 'merge-freeze' <<<"$labels"; then + frozen=$(gh issue list --label 'merge-freeze' --state open --json number,title \ + -q '.[] | "#\(.number) \(.title)"' | sed -n '1,5p') + if [ -n "$frozen" ]; then + reason+="Open 'merge-freeze' issue(s): $(tr '\n' ';' <<<"$frozen") " + fi + fi + + # --- Signal 2: a GHES version not yet on the default branch ------ + # Contents are read at the pinned commit, not at a branch name. + # These reads fail closed: a transient API error that looked like + # "no GHES directories" would silently drop the signal, so only a + # genuine 404 (the directory does not exist at that ref) becomes an + # empty listing. Anything else holds the merge. + errors=$(mktemp) + + ghes_dirs() { + local ref="$1" dir="$2" listing + if listing=$(gh api "repos/$GH_REPO/contents/$dir?ref=$ref" -q '.[].name' 2>"$errors"); then + grep '^ghes-' <<<"$listing" | sort -u || true + return 0 + fi + grep -q 'HTTP 404' "$errors" && return 0 + cat "$errors" >&2 + return 1 + } + + hold_unverifiable() { + echo "::error::Could not read $1 at $2 to check for a GHES release window; holding." + echo "status=frozen" >>"$GITHUB_OUTPUT" + echo "detail=Could not verify the GHES release state for $1 at $2" >>"$GITHUB_OUTPUT" + exit 0 + } + + if ! ghes_on_default=$(ghes_dirs "$DEFAULT_BRANCH" descriptions); then + hold_unverifiable descriptions "$DEFAULT_BRANCH" + fi + + for sha in "$SHA_30" "$SHA_31"; do + [ -n "$sha" ] || continue + for dir in descriptions descriptions-next; do + if ! on_pr=$(ghes_dirs "$sha" "$dir"); then + hold_unverifiable "$dir" "${sha:0:7}" + fi + [ -n "$on_pr" ] || continue + new=$(comm -13 <(echo "$ghes_on_default") <(echo "$on_pr") | tr '\n' ' ') + if [ -n "${new// /}" ]; then + reason+="${sha:0:7}/$dir introduces new GHES version(s): $new " + fi + done + done + + if [ -n "$reason" ]; then + echo "status=frozen" >>"$GITHUB_OUTPUT" + echo "detail=$reason" >>"$GITHUB_OUTPUT" + echo "::warning::Merge freeze in effect, holding. $reason" + else + echo "status=clear" >>"$GITHUB_OUTPUT" + echo "No merge freeze detected." + fi + + - name: Checkout repository (blobless) + if: steps.checks.outputs.status == 'green' && steps.freeze.outputs.status == 'clear' + uses: actions/checkout@v4 + with: + # Pinned like the tools checkout so the comparison baseline is + # provably the default branch, whatever ref a manual run was started + # from. + ref: ${{ github.event.repository.default_branch }} + path: repo + # Only used to read which files a candidate changes, which the API + # cannot answer for pull requests this large. A blobless fetch keeps + # this off the ~4.6 GB of history; blobs are fetched lazily. + filter: blob:none + fetch-depth: 0 + + - name: Fetch and pin candidate commits + id: pin + if: steps.checks.outputs.status == 'green' && steps.freeze.outputs.status == 'clear' + working-directory: repo + env: + SHA_30: ${{ steps.select.outputs.sha_30 }} + SHA_31: ${{ steps.select.outputs.sha_31 }} + PR_30: ${{ steps.select.outputs.pr_30 }} + PR_31: ${{ steps.select.outputs.pr_31 }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + + # `refs/pull//head` is fetched rather than the head branch so the + # commit selected earlier is obtained by SHA. If the branch has since + # moved, that object is simply not present and the run stops instead + # of validating one commit and merging another. + for pair in "$PR_30:$SHA_30" "$PR_31:$SHA_31"; do + pr="${pair%%:*}"; sha="${pair#*:}" + [ -n "$pr" ] && [ -n "$sha" ] || continue + + git fetch --no-tags --filter=blob:none origin "refs/pull/$pr/head" + + if ! git cat-file -e "$sha^{commit}" 2>/dev/null; then + echo "::notice::#$pr no longer resolves to $sha; stopping this run." + echo "status=stale" >>"$GITHUB_OUTPUT" + exit 0 + fi + + # A merge is only allowed to bring in description content. A pull + # request that touches workflows, scripts or anything else is left + # for a human, even when it carries the right title and author. + # This gate is about *what kind of file* changed, not about whether + # the description content itself is compatible. + # + # Evaluated here, before anything is written to the pull request, + # so an out-of-scope candidate never collects a comment promising a + # merge that will not happen. Both the commit and the local + # `origin/$DEFAULT_BRANCH` are fixed for the rest of the run, so + # this answer cannot change before the merge step. + base_commit=$(git merge-base "origin/$DEFAULT_BRANCH" "$sha") + # Captured before filtering so a failed `git diff` aborts the step + # instead of looking like "nothing outside the descriptions". + changed=$(git diff --name-only "$base_commit" "$sha") + # `sed -n` rather than `head`: it consumes all input, so the + # producer is never killed by SIGPIPE under `pipefail`. + outside=$(grep -v -E '^descriptions(-next)?/' <<<"$changed" | sed -n '1,10p' || true) + if [ -n "$outside" ]; then + echo "::notice::#$pr changes files outside the description directories; stopping." + echo "status=out-of-scope" >>"$GITHUB_OUTPUT" + echo "detail=#$pr changes $(tr '\n' ' ' <<<"$outside")" >>"$GITHUB_OUTPUT" + exit 0 + fi + + # Kept whole: a truncated inventory would omit files and the + # total line, which is the part worth reading. These updates + # routinely touch dozens of files but only a few kilobytes of + # stat, so this stays far inside the comment size limit. The + # guard below is for the pathological case only. + git diff --stat "$base_commit" "$sha" >"summary-$pr.txt" + if [ "$(wc -c <"summary-$pr.txt")" -gt 60000 ]; then + echo "::notice::#$pr has an unusually large change summary; truncating." + # `sed -n` rather than `head`: it consumes all input, so the + # producer is never killed by SIGPIPE under `pipefail`. + sed -n '1,500p' "summary-$pr.txt" >"summary-$pr.trimmed" + printf '\n... truncated; see the pull request files tab.\n' \ + >>"summary-$pr.trimmed" + mv "summary-$pr.trimmed" "summary-$pr.txt" + fi + done + + echo "status=pinned" >>"$GITHUB_OUTPUT" + + - name: Post change summary to pull requests + # A dry run is for confirming selection, so it stays read-only apart + # from the run summary. Posting here would also leave a comment + # carrying this commit's marker, which the later real run would treat + # as "already summarised" and skip. + if: steps.pin.outputs.status == 'pinned' && inputs.dry_run != true + working-directory: repo + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + SHA_30: ${{ steps.select.outputs.sha_30 }} + SHA_31: ${{ steps.select.outputs.sha_31 }} + PR_30: ${{ steps.select.outputs.pr_30 }} + PR_31: ${{ steps.select.outputs.pr_31 }} + run: | + set -euo pipefail + + # Release notes are generated from pull request descriptions, so an + # auto-merged pull request would otherwise land with no record of + # what the automation did. This is an inventory of the changed files, + # not a compatibility judgement: see the header for where + # compatibility review happens. + for pair in "$PR_30:$SHA_30" "$PR_31:$SHA_31"; do + pr="${pair%%:*}"; sha="${pair#*:}" + [ -n "$pr" ] && [ -n "$sha" ] || continue + + # The marker is per commit, so re-running for the same head does + # not add a duplicate comment. Comments are captured before + # matching so an early exiting `grep -q` cannot make `gh` look + # failed under `pipefail`. + marker="" + comments=$(gh pr view "$pr" --json comments -q '.comments[].body') + if grep -qF "$marker" <<<"$comments"; then + echo "Summary for $sha already posted on #$pr." + continue + fi + + { + echo "$marker" + echo "## Automated change summary" + echo + echo "Auto-merging \`${sha:0:7}\`, which changes these files:" + echo + echo '```' + cat "summary-$pr.txt" + echo '```' + echo + echo "_Posted by the auto-merge workflow ([run](${GITHUB_SERVER_URL}/${GH_REPO}/actions/runs/${GITHUB_RUN_ID}))._" + } >comment.md + + # No fallback: the summary is part of the merge contract, so a + # failure here must stop the run before the merge step. The + # per-commit marker makes a later retry safe. + gh pr comment "$pr" --body-file comment.md + done + + - name: Merge and close superseded pull requests + id: merge + if: steps.pin.outputs.status == 'pinned' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + SHA_30: ${{ steps.select.outputs.sha_30 }} + SHA_31: ${{ steps.select.outputs.sha_31 }} + PR_30: ${{ steps.select.outputs.pr_30 }} + PR_31: ${{ steps.select.outputs.pr_31 }} + SUPERSEDED_30: ${{ steps.select.outputs.superseded_30 }} + SUPERSEDED_31: ${{ steps.select.outputs.superseded_31 }} + DRY_RUN: ${{ inputs.dry_run }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + + # Titles come from the selection script so there is a single source + # of truth for what this workflow is allowed to touch, and each + # group is matched against its own exact title rather than against + # "either of the two". + selector="$GITHUB_WORKSPACE/tools/.github/scripts/select_openapi_prs.py" + title_30=$(python3 "$selector" --title 30) + title_31=$(python3 "$selector" --title 31) + + title_for() { + case "$1" in + 30) printf '%s\n' "$title_30" ;; + 31) printf '%s\n' "$title_31" ;; + esac + } + + candidates='' + for entry in "30:$PR_30:$SHA_30" "31:$PR_31:$SHA_31"; do + rest="${entry#*:}" + pr="${rest%%:*}"; sha="${rest#*:}" + [ -n "$pr" ] && [ -n "$sha" ] || continue + candidates+="$entry " + done + + # Confirm every candidate is still exactly what was validated: the + # same commit, still open, still authored by the description bot, + # still titled as a description update, and still targeting the + # default branch. Any mismatch means the pull request changed + # mid-run, so nothing is merged or closed. + for entry in $candidates; do + suffix="${entry%%:*}"; rest="${entry#*:}" + pr="${rest%%:*}"; sha="${rest#*:}" + meta=$(gh pr view "$pr" \ + --json headRefOid,state,author,title,baseRefName \ + -q '[.headRefOid, .state, .author.login, .baseRefName, .title] | @tsv') + IFS=$'\t' read -r current state author base title <<<"$meta" + + if [ "$current" != "$sha" ]; then + echo "::notice::#$pr moved from $sha to $current since validation; stopping." + echo "status=stale" >>"$GITHUB_OUTPUT" + echo "detail=#$pr moved during the run" >>"$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$state" != 'OPEN' ] || [ "$author" != "$BOT_LOGIN" ] \ + || [ "$base" != "$DEFAULT_BRANCH" ] \ + || [ "$title" != "$(title_for "$suffix")" ]; then + echo "::notice::#$pr is no longer an open $BOT_LOGIN description update targeting $DEFAULT_BRANCH; stopping." + echo "status=stale" >>"$GITHUB_OUTPUT" + echo "detail=#$pr changed during the run" >>"$GITHUB_OUTPUT" + exit 0 + fi + done + + if [ "$DRY_RUN" = "true" ]; then + echo "Dry run: would merge ${candidates:-none}, close: ${SUPERSEDED_30:-} ${SUPERSEDED_31:-}" + echo "status=dry-run" >>"$GITHUB_OUTPUT" + exit 0 + fi + + # Merged through the pull request API rather than by pushing the + # default branch: the branch requires a pull request, so a direct + # push is refused, and merging the pull request is the path that + # satisfies that rule without needing an exemption. + # + # `--match-head-commit` makes the server reject the merge unless the + # head is still the validated commit. That check and the merge are + # one atomic operation, so unlike a read followed by a write there is + # no window in which the head can move in between. It also makes the + # retry below safe: a retry can only ever merge the same commit. + merge_pr() { + pr="$1"; sha="$2" + for attempt in 1 2 3 4 5; do + if gh pr merge "$pr" --merge --match-head-commit "$sha"; then + return 0 + fi + + # These pull requests are large enough that the call can time out + # at the gateway while the merge still completes, so the recorded + # state is the authority here, not the exit code. + if [ "$(gh pr view "$pr" --json merged -q .merged 2>/dev/null)" = 'true' ]; then + echo "#$pr is merged; the call that performed it did not report back." + return 0 + fi + + if [ "$attempt" -lt 5 ]; then + echo "::notice::Merge attempt $attempt for #$pr did not succeed; retrying." + sleep $((attempt * 15)) + fi + done + return 1 + } + + merged='' + merge_failed='' + # The two versions are independent: they touch disjoint trees and are + # selected and superseded separately, so one failing must not change + # what happens to the other. Each version that merges is recorded so + # its superseded pull requests are still closed below. + merged_groups='' + for entry in $candidates; do + suffix="${entry%%:*}"; rest="${entry#*:}" + pr="${rest%%:*}"; sha="${rest#*:}" + + if merge_pr "$pr" "$sha"; then + merged+="#$pr " + merged_groups+="$suffix " + else + merge_failed+="#$pr " + echo "::error::#$pr could not be merged." + fi + done + + # Recorded immediately after the merges: everything below can fail, + # and none of it may hide what already landed. + if [ -n "$merge_failed" ]; then + echo "status=merge-failed" >>"$GITHUB_OUTPUT" + echo "detail=${merged:+merged ${merged% }, but }${merge_failed% } could not be merged" >>"$GITHUB_OUTPUT" + else + echo "status=merged" >>"$GITHUB_OUTPUT" + echo "detail=${merged% }" >>"$GITHUB_OUTPUT" + fi + + # Re-read current state immediately before closing. Selection + # happened minutes earlier, and a pull request can be retitled, + # retargeted, transferred or closed in between. A pull request is + # only closed when it is still open, still authored by the + # description bot, still targeting the default branch, and still + # carries the exact title it was superseded under. + close_failed='' + close_superseded() { + expected="$1"; shift + for pr in "$@"; do + # A failed read is treated exactly like a failed close: the pull + # request is still open and still needs a human. + if ! meta=$(gh pr view "$pr" --json title,state,author,baseRefName \ + -q '[.state, .author.login, .baseRefName, .title] | @tsv'); then + close_failed+="#$pr " + continue + fi + IFS=$'\t' read -r state author base title <<<"$meta" + + if [ "$state" != 'OPEN' ] || [ "$author" != "$BOT_LOGIN" ] \ + || [ "$base" != "$DEFAULT_BRANCH" ] || [ "$title" != "$expected" ]; then + echo "::notice::Leaving #$pr open: it no longer matches the pull request that was superseded." + continue + fi + + gh pr close "$pr" \ + --comment "Superseded by the newer OpenAPI description update(s) ${merged% }. Closing automatically." \ + || close_failed+="#$pr " + done + } + + # Only for versions that actually merged. Closing them is what stops + # the next run from selecting a stale snapshot: once the newest pull + # request is merged and therefore closed, an older one left open + # becomes the newest open one, and merging that would put an earlier + # description back on the default branch. A version whose merge + # failed keeps its superseded pull requests, because its newest one + # is still open and still the right candidate. + for suffix in $merged_groups; do + case "$suffix" in + 30) close_superseded "$title_30" ${SUPERSEDED_30:-} ;; + 31) close_superseded "$title_31" ${SUPERSEDED_31:-} ;; + esac + done + + # A superseded pull request left open would be picked up as a + # candidate by the next run, so this must not pass silently. + if [ -n "$close_failed" ]; then + echo "close_failed=${close_failed% }" >>"$GITHUB_OUTPUT" + echo "::error::Failed to close superseded pull request(s): $close_failed" + fi + + if [ -n "$close_failed" ] || [ -n "$merge_failed" ]; then + exit 1 + fi + + - name: Report outcome + # Runs on every completed path so the outcome is decided in exactly one + # place and the job summary and the notification cannot disagree. + if: ${{ !cancelled() }} + env: + CHATTERBOX_URL: ${{ secrets.CHATTERBOX_URL }} + CHATTERBOX_TOKEN: ${{ secrets.CHATTERBOX_TOKEN }} + GH_REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SELECT_FOUND: ${{ steps.select.outputs.found }} + CHECKS_STATUS: ${{ steps.checks.outputs.status }} + CHECKS_DETAIL: ${{ steps.checks.outputs.detail }} + FREEZE_STATUS: ${{ steps.freeze.outputs.status }} + FREEZE_DETAIL: ${{ steps.freeze.outputs.detail }} + PIN_STATUS: ${{ steps.pin.outputs.status }} + PIN_DETAIL: ${{ steps.pin.outputs.detail }} + MERGE_STATUS: ${{ steps.merge.outputs.status }} + MERGE_DETAIL: ${{ steps.merge.outputs.detail }} + MERGE_CLOSE_FAILED: ${{ steps.merge.outputs.close_failed }} + run: | + set -euo pipefail + + # `notify` marks the outcomes a person needs to see. The schedule runs + # twelve times a day, and a freeze or a red check persists across many + # of those runs, so notifying on them would produce a stream of + # identical messages that trains people to ignore the channel. Those + # outcomes are recorded in the job summary instead. Lint failures + # already have their own notification. + notify='false' + + if [ "$MERGE_STATUS" = 'merge-failed' ]; then + icon=':x:' + headline='Auto-merge failed' + detail="The workflow $MERGE_DETAIL.${MERGE_CLOSE_FAILED:+ Superseded pull request(s) $MERGE_CLOSE_FAILED could not be closed either.}" + notify='true' + elif [ -n "$MERGE_CLOSE_FAILED" ]; then + icon=':warning:' + headline='Auto-merge needs follow-up' + detail="Merged $MERGE_DETAIL, but superseded pull request(s) $MERGE_CLOSE_FAILED could not be closed. Close them manually, otherwise the next run treats one of them as a candidate." + notify='true' + elif [ "$MERGE_STATUS" = 'merged' ]; then + icon=':white_check_mark:' + headline='Auto-merged OpenAPI description updates' + detail="Merged $MERGE_DETAIL" + notify='true' + elif [ "$MERGE_STATUS" = 'dry-run' ]; then + icon=':information_source:' + headline='Auto-merge dry run' + detail='Nothing was merged or closed.' + elif [ "$CHECKS_STATUS" = 'blocked' ]; then + icon=':warning:' + headline='Auto-merge did not proceed' + detail="Required checks were not green: $CHECKS_DETAIL" + elif [ "$FREEZE_STATUS" = 'frozen' ]; then + icon=':snowflake:' + headline='Auto-merge held' + detail="A merge freeze was detected: $FREEZE_DETAIL" + elif [ "$PIN_STATUS" = 'out-of-scope' ]; then + icon=':warning:' + headline='Auto-merge did not proceed' + detail="A candidate changed files outside the description directories: $PIN_DETAIL" + notify='true' + elif [ "$PIN_STATUS" = 'stale' ] || [ "$MERGE_STATUS" = 'stale' ]; then + icon=':information_source:' + headline='Auto-merge did not proceed' + detail='A candidate pull request was updated mid-run, so nothing was merged.' + elif [ "$SELECT_FOUND" = 'false' ]; then + icon=':information_source:' + headline='Nothing to do' + detail='No open OpenAPI description update pull requests.' + else + icon=':x:' + headline='Auto-merge failed' + detail='The workflow failed before merging. Review the failed step above.' + notify='true' + fi + + # Written before the notification is attempted, so the run always + # carries the outcome even if posting it fails. + { + echo "## $headline" + echo + echo "$detail" + } >>"$GITHUB_STEP_SUMMARY" + + echo "$headline: $detail" + [ "$notify" = 'true' ] || exit 0 + + if [ -z "${CHATTERBOX_URL:-}" ] || [ -z "${CHATTERBOX_TOKEN:-}" ]; then + echo "CHATTERBOX_URL or CHATTERBOX_TOKEN is not configured; skipping notification." + exit 0 + fi + + message=$(printf '%s\n' \ + "$icon $headline in ${GH_REPO}." \ + "• ${detail}" \ + "• Run: ${RUN_URL}") + + curl --fail --silent --show-error \ + -X POST \ + -u "${CHATTERBOX_TOKEN}:" \ + "${CHATTERBOX_URL%/}/topics/%23api-platform" \ + --data "$message" diff --git a/.github/workflows/scripts-tests.yml b/.github/workflows/scripts-tests.yml new file mode 100644 index 0000000000..c52b1b2e4f --- /dev/null +++ b/.github/workflows/scripts-tests.yml @@ -0,0 +1,29 @@ +--- +name: Test workflow scripts +permissions: + contents: read + +on: + push: + paths: + - '.github/scripts/**' + - '.github/workflows/scripts-tests.yml' + pull_request: + paths: + - '.github/scripts/**' + - '.github/workflows/scripts-tests.yml' + workflow_dispatch: + +jobs: + unittest: + name: Unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + name: Install Python + with: + python-version: '3.12' + - run: python3 -m unittest discover --start-directory .github/scripts --verbose + name: Run script tests diff --git a/.gitignore b/.gitignore index 40b878db5b..4b7a81db08 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -node_modules/ \ No newline at end of file +node_modules/ +__pycache__/