Implement results issues command

`kci-dev results issues` command utilities:
- Fetch KCIDB issues from the dashboard
- Get new issues for checkout with `--new` option
- Get failed/inconclusive builds and boots without
any associated issues with `--missing` option

Signed-off-by: Jeny Sadadia <[email protected]>

Author: Jeny Sadadia

kcidev/libs/dashboard.py

@@ -285,23 +285,24 @@ def dashboard_fetch_hardware_tests(name, origin, use_json):
     )
 
 
-def dashboard_fetch_build_issues(build_id, use_json):
+def dashboard_fetch_build_issues(build_id, use_json, error_verbose):
     endpoint = f"build/{build_id}/issues"
     logging.info(f"Fetching build issues for build ID: {build_id}")
-    return dashboard_api_fetch(endpoint, {}, use_json)
+    return dashboard_api_fetch(endpoint, {}, use_json, error_verbose=error_verbose)
 
 
-def dashboard_fetch_boot_issues(test_id, use_json):
+def dashboard_fetch_boot_issues(test_id, use_json, error_verbose):
     endpoint = f"test/{test_id}/issues"
     logging.info(f"Fetching test issues for test ID: {test_id}")
-    return dashboard_api_fetch(endpoint, {}, use_json)
+    return dashboard_api_fetch(endpoint, {}, use_json, error_verbose=error_verbose)
 
 
 def dashboard_fetch_issue_list(origin, days, use_json):
     params = {
-        "filter_origin": origin,
         "interval_in_days": days,
     }
+    if origin:
+        params["filter_origin"] = origin
     logging.info(f"Fetching issue list for origin: {origin}")
     return dashboard_api_fetch("issue/", params, use_json)
 

kcidev/subcommands/results/__init__.py

@@ -6,7 +6,7 @@
 import click
 from tabulate import tabulate
 
-from kcidev.libs.common import kci_msg, kci_msg_green, kci_msg_red
+from kcidev.libs.common import kci_msg, kci_msg_bold, kci_msg_green, kci_msg_red
 from kcidev.libs.dashboard import (
     dashboard_fetch_boot_issues,
     dashboard_fetch_boots,
@@ -16,6 +16,7 @@
     dashboard_fetch_commits_history,
     dashboard_fetch_issue,
     dashboard_fetch_issue_builds,
+    dashboard_fetch_issue_list,
     dashboard_fetch_issue_tests,
     dashboard_fetch_issues_extra,
     dashboard_fetch_summary,
@@ -41,6 +42,7 @@
     cmd_tests,
     print_issue,
     print_issues,
+    print_missing_data,
 )
 
 
@@ -473,21 +475,22 @@ def print_stats(data, headers, max_col_width, table_fmt):
     )
 
 
-def get_new_issues(origin, tree_name, giturl, branch, commit, arch):
+def get_new_issues(origin, tree_name, giturl, branch, commit, arch, use_json=False):
     """Get new KCIDB issue for a checkout"""
     try:
+        kci_msg_green(f"{tree_name}/{branch}:{commit}")
         data = dashboard_fetch_summary(origin, giturl, branch, commit, arch, True)
         tree_summary = data["summary"]
         items = ["builds", "boots", "tests"]
-        all_issues = []
         new_issues = []
+        issue_id_version = []
         for item in items:
             for i in tree_summary[item]["issues"]:
-                all_issues.append([i["id"], i["version"]])
-        if not all_issues:
-            kci_msg_red(f"{tree_name}/{branch}:{commit} No issues found")
+                issue_id_version.append([i["id"], i["version"]])
+        if not issue_id_version:
+            kci_msg("No issues found")
             return
-        issue_extras = dashboard_fetch_issues_extra(all_issues, True)["issues"]
+        issue_extras = dashboard_fetch_issues_extra(issue_id_version, True)["issues"]
         for issue_id, extras in issue_extras.items():
             first_incident = extras.get("first_incident")
             if first_incident:
@@ -496,13 +499,14 @@ def get_new_issues(origin, tree_name, giturl, branch, commit, arch):
                         first_incident["git_commit_hash"] == commit,
                         first_incident["git_repository_url"] == giturl,
                         first_incident["git_repository_branch"] == branch,
+                        first_incident["tree_name"] == tree_name,
                     ]
                 ):
-                    new_issues.append(issue_id)
+                    new_issues.append(dashboard_fetch_issue(issue_id, False))
         if not new_issues:
-            kci_msg_red(f"{tree_name}/{branch}:{commit} No new issues found")
+            kci_msg("No new issues found")
         else:
-            kci_msg_green(f"{tree_name}/{branch}:{commit} New issues: {new_issues}")
+            print_issues(new_issues)
     except click.ClickException as e:
         print("Exception:", e.message)
 
@@ -682,6 +686,215 @@ def detect(
             print_stats(stats, headers, max_col_width, table_fmt)
 
 
+def get_missing_issue_items(
+    ctx, origin, item_type, giturl, branch, commit, tree_name, arch
+):
+    """Get information of failed or inconclusive builds/boots for which KCIDB
+    issues don't exist"""
+    try:
+        if item_type == "builds":
+            results_cmd = builds
+            dashboard_func = dashboard_fetch_build_issues
+        elif item_type == "boots":
+            results_cmd = boots
+            dashboard_func = dashboard_fetch_boot_issues
+        else:
+            kci_msg_red("Please specify 'builds' or 'boots' as items type")
+            return []
+
+        dashboard_items = ctx.invoke(
+            results_cmd,
+            origin=origin,
+            giturl=giturl,
+            branch=branch,
+            commit=commit,
+            status="all",
+            count=True,
+            verbose=False,
+            arch=arch,
+        )
+
+        missing_ids = []
+        for item in dashboard_items:
+            # Exclude passed builds/boots
+            if item["status"] == "PASS":
+                continue
+            item_id = item["id"]
+            try:
+                _ = dashboard_func(item_id, False, error_verbose=False)
+            except click.ClickException as e:
+                if "No issues" in e.message:
+                    missing_ids.append({item_id: item["status"]})
+        if missing_ids:
+            return [f"{tree_name}/{branch}", commit, missing_ids]
+        return []
+    except click.Abort:
+        kci_msg_red(
+            f"{tree_name}/{branch}: Aborted while fetching dashboard builds/boots"
+        )
+        return []
+    except click.ClickException as e:
+        kci_msg_red(f"{tree_name}/{branch}: {e.message}")
+        return []
+
+
+@results.command(
+    name="issues",
+    help="""Fetch KCIDB issues from the dashboard
+
+\b
+Examples:
+  # Get issues
+  kci-dev results issues --origin <origin> --days <number-of-days>
+  # Get new issues for all checkouts
+  kci-dev issues --new --days <number-of-days> --origin <origin>
+  # Get new issues for a checkout
+  kci-dev issues --new --giturl <git-url> --branch <git-branch> --commit <commit> --origin <origin>
+  # Get failed or inconclusive builds and boots without any issue for all checkouts
+  kci-dev issues --missing
+  # Get failed or inconclusive builds and boots without any issue for specific checkout
+  kci-dev issues --missing --giturl <git-url> --branch <git-branch> --commit <commit> --origin <origin>
+  # Get failed/inconclusive builds without issues
+  kci-dev issues --missing --builds
+  # Get failed/inconclusive boots without issues
+  kci-dev issues --missing --boots
+""",
+)
+@click.option(
+    "--origin",
+    help="Select KCIDB origin",
+    default="maestro",
+)
+@click.option(
+    "--days",
+    help="Provide a period of time in days to get results for",
+    type=int,
+    default="5",
+)
+@click.option(
+    "--new",
+    is_flag=True,
+    help="Fetch new KCIDB issues for a checkout",
+)
+@click.option(
+    "--giturl",
+    help="Git URL of kernel tree",
+)
+@click.option(
+    "--branch",
+    help="Branch to get results for",
+)
+@click.option(
+    "--commit",
+    help="Commit or tag to get results for",
+)
+@click.option(
+    "--git-folder",
+    help="Path of git repository folder",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+)
+@click.option(
+    "--latest",
+    is_flag=True,
+    help="Select latest results available",
+)
+@click.option(
+    "--missing",
+    is_flag=True,
+    help="List all failed/incomplete builds and boots without any issue associated to them",
+)
+@click.option(
+    "--builds",
+    is_flag=True,
+    help="The option can be used along with '--missing' to list only builds",
+)
+@click.option(
+    "--boots",
+    is_flag=True,
+    help="The option can be used along with '--missing' to list only boots",
+)
+@click.option("--arch", help="Filter by arch")
+@click.pass_context
+@results_display_options
+def issues(
+    ctx,
+    origin,
+    days,
+    new,
+    giturl,
+    branch,
+    commit,
+    use_json,
+    latest,
+    git_folder,
+    arch,
+    missing,
+    builds,
+    boots,
+):
+    """Issues command handler"""
+    if not new and not missing:
+        data = dashboard_fetch_issue_list(origin, days, use_json)
+        print_issues(data)
+    if new:
+        if not all([giturl, branch, commit]):
+            kci_msg("Fetching new issues for all checkouts...")
+            trees_list = ctx.invoke(trees, origin=origin, days=days, verbose=False)
+            for tree in trees_list:
+                giturl = tree["git_repository_url"]
+                branch = tree["git_repository_branch"]
+                commit = tree["git_commit_hash"]
+                tree_name = tree["tree_name"]
+                get_new_issues(
+                    origin, tree_name, giturl, branch, commit, arch, use_json
+                )
+            return
+        kci_msg("Fetching new issues for the checkout...")
+        giturl, branch, commit = set_giturl_branch_commit(
+            origin, giturl, branch, commit, latest, git_folder
+        )
+        tree_name = get_tree_name(origin, giturl, branch)
+        get_new_issues(origin, tree_name, giturl, branch, commit, arch, use_json)
+    if missing:
+        if builds:
+            item_types = ["builds"]
+        elif boots:
+            item_types = ["boots"]
+        else:
+            item_types = ["builds", "boots"]
+
+        if all([giturl, branch, commit]):
+            tree_name = get_tree_name(origin, giturl, branch)
+            for item_type in item_types:
+                final_stats = []
+                kci_msg_green(f"Fetching data for {item_type}...")
+                stats = get_missing_issue_items(
+                    ctx, origin, item_type, giturl, branch, commit, tree_name, arch
+                )
+                if stats:
+                    final_stats.append(stats)
+                if final_stats:
+                    print_missing_data(item_type, final_stats)
+
+        else:
+            trees_list = ctx.invoke(trees, origin=origin, days=days, verbose=False)
+            for item_type in item_types:
+                kci_msg_green(f"Fetching data for {item_type}...")
+                final_stats = []
+                for tree in trees_list:
+                    giturl = tree["git_repository_url"]
+                    branch = tree["git_repository_branch"]
+                    commit = tree["git_commit_hash"]
+                    tree_name = tree["tree_name"]
+                    stats = get_missing_issue_items(
+                        ctx, origin, item_type, giturl, branch, commit, tree_name, arch
+                    )
+                    if stats:
+                        final_stats.append(stats)
+                if final_stats:
+                    print_missing_data(item_type, final_stats)
+
+
 @results.command(
     name="issue",
     help="""Fetch KCIDB issue matching provided ID.

kcidev/subcommands/results/parser.py

@@ -986,3 +986,60 @@ def print_issue(issue):
             kci_msg_nonl("  First incident seen on ")
             kci_msg_bold(tree_branch)
     kci_msg("")
+
+
+def print_issues(data):
+    """Print a list of issues with formatting
+    data (dict): provide dictionary with 'issues' and 'extras' keys"""
+    parsed = urlparse(DASHBOARD_API)
+    dashboard_url = f"{parsed.scheme}://{parsed.netloc}"
+
+    if not data.get("issues") or not data.get("extras"):
+        kci_err("Please provide dictionary with 'issues' and 'extras' keys")
+        return
+
+    issues = data["issues"]
+    extras = data["extras"]
+    for issue in issues:
+        print_issue_information(issue, dashboard_url)
+
+        extra = extras.get(issue.get("id"))
+        if extra:
+            tree_branch = (
+                f'{extra.get("tree_name")}/{extra.get("git_repository_branch")}'
+            )
+            kci_msg_nonl("  First incident seen on ")
+            kci_msg_bold(tree_branch)
+        kci_msg("")
+
+
+def print_missing_data(item_type, data):
+    """Print builds/tests for which KCIDB issues are missing"""
+    from collections import defaultdict
+
+    tree_groups = defaultdict(list)
+
+    for row in data:
+        try:
+            tree_branch = row[0]
+            commit = row[1]
+            missing_ids = row[2]
+        except IndexError:
+            kci_msg_red("Failed to extract data for list view")
+            continue
+        tree_groups[tree_branch].append({"commit": commit, "missing_ids": missing_ids})
+
+    parsed = urlparse(DASHBOARD_API)
+    dashboard_url = f"{parsed.scheme}://{parsed.netloc}"
+    if item_type == "builds":
+        endpoint = "build"
+    else:
+        endpoint = "test"
+    for tree_branch, commits in tree_groups.items():
+        kci_msg_bold(f"{tree_branch}: ")
+        for commit in commits:
+            kci_msg(f"  - Commit:{commit['commit']}")
+            for item in missing_ids:
+                for item_id, status in item.items():
+                    kci_msg_nonl(f"  {dashboard_url}/{endpoint}/{item_id}  status: ")
+                    kci_msg_red(f"{status}")