|
| 1 | +#!/usr/bin/env python |
| 2 | + |
| 3 | +import argparse |
| 4 | +import io |
| 5 | +import json |
| 6 | +import os |
| 7 | +import re |
| 8 | +from datetime import datetime |
| 9 | +from textwrap import dedent |
| 10 | +from typing import cast |
| 11 | +from urllib.request import urlopen |
| 12 | + |
| 13 | +import diskcache |
| 14 | +import github |
| 15 | +import koji |
| 16 | +import requests |
| 17 | +from github.Commit import Commit |
| 18 | +from github.GithubException import BadCredentialsException |
| 19 | +from github.PullRequest import PullRequest |
| 20 | + |
| 21 | + |
| 22 | +def print_header(out): |
| 23 | + print(dedent(''' |
| 24 | + <!DOCTYPE html> |
| 25 | + <html lang="en"> |
| 26 | + <head> |
| 27 | + <meta charset="UTF-8"> |
| 28 | + <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| 29 | + <title>XCP-ng Package Update</title> |
| 30 | + <script src="https://cdn.tailwindcss.com"></script> |
| 31 | + </head> |
| 32 | + <body class="bg-gray-400 text-center"> |
| 33 | + '''), file=out) |
| 34 | + |
| 35 | +def print_plane_warning(out): |
| 36 | + print(dedent(''' |
| 37 | + <div class="px-3 py-3"> |
| 38 | + <div class="bg-orange-100 border-l-4 border-orange-500 text-orange-700 p-4" role="alert"> |
| 39 | + <p class="font-bold">Plane malfunction</p> |
| 40 | + <p>The issues could not be retrieved from plane.</p> |
| 41 | + </div> |
| 42 | + </div>'''), file=out) |
| 43 | + |
| 44 | +def print_github_warning(out): |
| 45 | + print(dedent(''' |
| 46 | + <div class="px-3 py-3"> |
| 47 | + <div class="bg-orange-100 border-l-4 border-orange-500 text-orange-700 p-4" role="alert"> |
| 48 | + <p class="font-bold">Github access problem</p> |
| 49 | + <p>The pull requests come from the cache and may not be up to date.</p> |
| 50 | + </div> |
| 51 | + </div>'''), file=out) |
| 52 | + |
| 53 | +def print_koji_error(out): |
| 54 | + print(dedent(''' |
| 55 | + <div class="px-3 py-3"> |
| 56 | + <div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative" role="alert"> |
| 57 | + <strong class="font-bold">Koji error!</strong> |
| 58 | + <span class="block sm:inline">The report can't be generated.</span> |
| 59 | + </div> |
| 60 | + </div>'''), file=out) |
| 61 | + |
| 62 | +def print_generic_error(out): |
| 63 | + print(dedent(''' |
| 64 | + <div class="px-3 py-3"> |
| 65 | + <div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative" role="alert"> |
| 66 | + <strong class="font-bold">Unknown error!</strong> |
| 67 | + <span class="block sm:inline">The report can't be generated.</span> |
| 68 | + </div> |
| 69 | + </div>'''), file=out) |
| 70 | + |
| 71 | +def print_footer(out, generated_info): |
| 72 | + now = datetime.now() |
| 73 | + print(dedent(f''' |
| 74 | + Last generated at {now}. {generated_info or ''} |
| 75 | + </body> |
| 76 | + </html> |
| 77 | + '''), file=out) |
| 78 | + |
| 79 | +def print_table_header(out, tag): |
| 80 | + print(dedent(f''' |
| 81 | + <div class="px-3 py-3"> |
| 82 | + <div class="relative overflow-x-auto shadow-md sm:rounded-lg"> |
| 83 | + <table class="table-fixed w-full text-sm text-left rtl:text-right text-gray-500 dark:text-gray-400"> |
| 84 | + <caption class="p-5 text-lg font-semibold text-left rtl:text-right text-gray-900 bg-white dark:text-white dark:bg-gray-800"> |
| 85 | + {tag} |
| 86 | + </caption> |
| 87 | + <thead class="text-xs text-gray-700 uppercase bg-gray-50 dark:bg-gray-700 dark:text-gray-400"> |
| 88 | + <tr> |
| 89 | + <th scope="col" class="px-6 py-3"> |
| 90 | + Build |
| 91 | + </th> |
| 92 | + <th scope="col" class="px-6 py-3"> |
| 93 | + Cards |
| 94 | + </th> |
| 95 | + <th scope="col" class="px-6 py-3"> |
| 96 | + Pull Requests |
| 97 | + </th> |
| 98 | + <th scope="col" class="px-6 py-3"> |
| 99 | + Built by |
| 100 | + </th> |
| 101 | + <th scope="col" class="px-6 py-3"> |
| 102 | + Maintained by |
| 103 | + </th> |
| 104 | + </tr> |
| 105 | + </thead> |
| 106 | + <tbody> |
| 107 | + '''), file=out) # nopep8 |
| 108 | + |
| 109 | +def print_table_footer(out): |
| 110 | + print(dedent(''' |
| 111 | + </tbody> |
| 112 | + </table> |
| 113 | + </div> |
| 114 | + </div> |
| 115 | + '''), file=out) |
| 116 | + |
| 117 | +def print_table_line(out, build, link, issues, built_by, prs: list[PullRequest], maintained_by): |
| 118 | + issues_content = '\n'.join([ |
| 119 | + f'''<li> |
| 120 | + <a class="font-medium text-blue-600 dark:text-blue-500 hover:underline" |
| 121 | + href="https://project.vates.tech/vates-global/browse/XCPNG-{i['sequence_id']}/">XCPNG-{i['sequence_id']} |
| 122 | + </a> |
| 123 | + </li>''' |
| 124 | + for i in issues |
| 125 | + ]) |
| 126 | + prs_content = '\n'.join([ |
| 127 | + f'''<li> |
| 128 | + <a class="font-medium text-blue-600 dark:text-blue-500 hover:underline" |
| 129 | + href="{pr.html_url}">{pr.title} #{pr.number} |
| 130 | + </a> |
| 131 | + </li>''' |
| 132 | + for pr in prs |
| 133 | + ]) |
| 134 | + print(f''' |
| 135 | + <tr class="odd:bg-white odd:dark:bg-gray-900 even:bg-gray-50 even:dark:bg-gray-800 border-b dark:border-gray-700 border-gray-200"> |
| 136 | + <th scope="row" class="px-6 py-4 font-medium text-gray-900 whitespace-nowrap dark:text-white"> |
| 137 | + <a class="font-medium text-blue-600 dark:text-blue-500 hover:underline" href="{link}">{build}</a> |
| 138 | + </th> |
| 139 | + <td class="px-6 py-4"> |
| 140 | + <ul> |
| 141 | + {issues_content} |
| 142 | + </ul> |
| 143 | + </td> |
| 144 | + <td class="px-6 py-4"> |
| 145 | + <ul> |
| 146 | + {prs_content} |
| 147 | + </ul> |
| 148 | + </td> |
| 149 | + <td class="px-6 py-4"> |
| 150 | + {built_by} |
| 151 | + </td> |
| 152 | + <td class="px-6 py-4"> |
| 153 | + {maintained_by if maintained_by is not None else ''} |
| 154 | + </td> |
| 155 | + </tr> |
| 156 | + ''', file=out) # nopep8 |
| 157 | + |
| 158 | +def parse_source(source: str) -> tuple[str, str]: |
| 159 | + groups = re.match(r'git\+https://github\.com/([\w-]+/[\w-]+)(|\.git)#([0-9a-f]{40})', source) |
| 160 | + assert groups is not None, "can't match the source to the expected github url" |
| 161 | + return (groups[1], groups[3]) |
| 162 | + |
| 163 | +def filter_issues(issues, urls): |
| 164 | + res = [] |
| 165 | + for issue in issues: |
| 166 | + for url in urls: |
| 167 | + url = url.strip('/') |
| 168 | + if f'href="{url}"' in issue['description_html'] or f'href="{url}/"' in issue['description_html']: |
| 169 | + res.append(issue) |
| 170 | + break |
| 171 | + return res |
| 172 | + |
| 173 | + |
| 174 | +TAG_ORDER = ['incoming', 'ci', 'testing', 'candidates', 'updates', 'base', 'lab'] |
| 175 | + |
| 176 | +def tag_priority(tag): |
| 177 | + # drop the version in the tag — v8.3-incoming -> incoming |
| 178 | + tag = tag.split('-')[-1] |
| 179 | + return TAG_ORDER.index(tag) |
| 180 | + |
| 181 | +def find_previous_build_commit(session, build_tag, build): |
| 182 | + """Find the previous build in an higher priority koji tag and return its commit.""" |
| 183 | + tagged = session.listTagged(build_tag, package=build['package_name'], inherit=True) |
| 184 | + tagged = sorted(tagged, key=lambda t: tag_priority(t['tag_name'])) |
| 185 | + build_tag_priority = tag_priority(build_tag) |
| 186 | + tagged = [t for t in tagged if tag_priority(t['tag_name']) > build_tag_priority] |
| 187 | + if not tagged: |
| 188 | + return None |
| 189 | + previous_build = session.getBuild(tagged[0]['build_id']) |
| 190 | + if not previous_build.get('source'): |
| 191 | + return None |
| 192 | + return parse_source(previous_build['source'])[1] |
| 193 | + |
| 194 | +def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]: |
| 195 | + """ |
| 196 | + List the commits in the range [start_sha,end_sha[. |
| 197 | +
|
| 198 | + Note: these are the commits listed by Github starting from start_sha up to end_sha excluded. |
| 199 | + A commit older that the end_sha commit and added by a merge commit won't appear in this list. |
| 200 | + """ |
| 201 | + cache_key = f'commits-{start_sha}-{end_sha}' |
| 202 | + if cache_key in CACHE: |
| 203 | + return cast(list[Commit], CACHE[cache_key]) |
| 204 | + commits = [] |
| 205 | + if gh: |
| 206 | + for commit in gh.get_repo(repo).get_commits(start_sha): |
| 207 | + if commit.sha == end_sha: |
| 208 | + break |
| 209 | + commits.append(commit) |
| 210 | + CACHE[cache_key] = commits |
| 211 | + return commits |
| 212 | + |
| 213 | +def find_pull_requests(gh, repo, start_sha, end_sha): |
| 214 | + """Find the pull requests for the commits in the [start_sha,end_sha[ range.""" |
| 215 | + prs = set() |
| 216 | + for commit in find_commits(gh, repo, start_sha, end_sha): |
| 217 | + cache_key = f'commit-prs-{commit.sha}' |
| 218 | + if cache_key in CACHE: |
| 219 | + prs.update(cast(list[PullRequest], CACHE[cache_key])) |
| 220 | + elif gh: |
| 221 | + commit_prs = list(commit.get_pulls()) |
| 222 | + CACHE[cache_key] = commit_prs |
| 223 | + prs.update(commit_prs) |
| 224 | + return sorted(prs, key=lambda p: p.number, reverse=True) |
| 225 | + |
| 226 | +parser = argparse.ArgumentParser(description='Generate a report of the packages in the pipe') |
| 227 | +parser.add_argument('output', nargs='?', help='Report output path', default='report.html') |
| 228 | +parser.add_argument('--generated-info', help="Add this message about the generation in the report") |
| 229 | +parser.add_argument( |
| 230 | + '--plane-token', help="The token used to access the plane api", default=os.environ.get('PLANE_TOKEN') |
| 231 | +) |
| 232 | +parser.add_argument( |
| 233 | + '--github-token', help="The token used to access the Github api", default=os.environ.get('GITHUB_TOKEN') |
| 234 | +) |
| 235 | +parser.add_argument('--cache', help="The cache path", default="/tmp/pkg_in_pipe.cache") |
| 236 | +args = parser.parse_args() |
| 237 | + |
| 238 | +CACHE = diskcache.Cache(args.cache) |
| 239 | + |
| 240 | +# load the issues from plane, so we can search for the plane card related to a build |
| 241 | +resp = requests.get( |
| 242 | + 'https://project.vates.tech/api/v1/workspaces/vates-global/projects/43438eec-1335-4fc2-8804-5a4c32f4932d/issues/', |
| 243 | + headers={'x-api-key': args.plane_token}, |
| 244 | +) |
| 245 | +issues = resp.json().get('results', []) |
| 246 | + |
| 247 | +# connect to github |
| 248 | +if args.github_token: |
| 249 | + gh = github.Github(auth=github.Auth.Token(args.github_token)) |
| 250 | + try: |
| 251 | + gh.get_repo('xcp-ng/xcp') # check that the token is valid |
| 252 | + except BadCredentialsException: |
| 253 | + gh = None |
| 254 | +else: |
| 255 | + gh = None |
| 256 | + |
| 257 | +# load the packages maintainers |
| 258 | +with urlopen('https://github.com/xcp-ng/xcp/raw/refs/heads/master/scripts/rpm_owners/packages.json') as f: |
| 259 | + PACKAGES = json.load(f) |
| 260 | + |
| 261 | +ok = True |
| 262 | +with open(args.output, 'w') as out: |
| 263 | + print_header(out) |
| 264 | + if not issues: |
| 265 | + print_plane_warning(out) |
| 266 | + if not gh: |
| 267 | + print_github_warning(out) |
| 268 | + tags = [f'v{v}-{p}' for v in ['8.2', '8.3'] for p in ['incoming', 'ci', 'testing', 'candidates', 'lab']] |
| 269 | + temp_out = io.StringIO() |
| 270 | + try: |
| 271 | + # open koji session |
| 272 | + config = koji.read_config("koji") |
| 273 | + session = koji.ClientSession('https://kojihub.xcp-ng.org', config) |
| 274 | + session.ssl_login(config['cert'], None, config['serverca']) |
| 275 | + for tag in tags: |
| 276 | + print_table_header(temp_out, tag) |
| 277 | + for tagged in sorted(session.listTagged(tag), key=lambda build: int(build['build_id']), reverse=True): |
| 278 | + build = session.getBuild(tagged['build_id']) |
| 279 | + prs: list[PullRequest] = [] |
| 280 | + maintained_by = None |
| 281 | + previous_build_sha = find_previous_build_commit(session, tag, build) |
| 282 | + if build['source'] is not None: |
| 283 | + (repo, sha) = parse_source(build['source']) |
| 284 | + prs = find_pull_requests(gh, repo, sha, previous_build_sha) |
| 285 | + maintained_by = PACKAGES.get(repo.split('/')[-1], {}).get('maintainer') |
| 286 | + build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={tagged["build_id"]}' |
| 287 | + build_issues = filter_issues(issues, [build_url] + [pr.html_url for pr in prs]) |
| 288 | + print_table_line( |
| 289 | + temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs, maintained_by |
| 290 | + ) |
| 291 | + print_table_footer(temp_out) |
| 292 | + out.write(temp_out.getvalue()) |
| 293 | + except koji.GenericError: |
| 294 | + print_koji_error(out) |
| 295 | + raise |
| 296 | + except Exception: |
| 297 | + print_generic_error(out) |
| 298 | + raise |
| 299 | + finally: |
| 300 | + print_footer(out, args.generated_info) |
0 commit comments