tc

Author: clee2000

torchci/clickhouse_queries/commit_jobs_query/params.json

@@ -1,12 +1,14 @@
 {
   "params": {
     "repo": "String",
-    "sha": "String"
+    "sha": "String",
+    "workflowId": "Int64"
   },
   "tests": [
     {
       "repo": "pytorch/pytorch",
-      "sha": "85df746892d9b0e87e7a5dfa78ef81a84aec6de0"
+      "sha": "85df746892d9b0e87e7a5dfa78ef81a84aec6de0",
+      "workflow_id": 0
     }
   ]
 }

torchci/clickhouse_queries/commit_jobs_query/query.sql

@@ -46,6 +46,10 @@ WITH job AS (
         AND workflow.event != 'workflow_run' -- Filter out workflow_run-triggered jobs, which have nothing to do with the SHA
         AND workflow.event != 'repository_dispatch' -- Filter out repository_dispatch-triggered jobs, which have nothing to do with the SHA
         AND workflow.id in (select id from materialized_views.workflow_run_by_head_sha where head_sha = {sha: String})
+        AND (
+            {workflowId: Int64} = 0
+            OR workflow.id = {workflowId: Int64} -- If a specific workflow ID is provided, filter by it
+        )
         AND job.id in (select id from materialized_views.workflow_job_by_head_sha where head_sha = {sha: String})
         AND workflow.repository. 'full_name' = {repo: String } --         UNION
         AND workflow.name != 'Upload test stats while running' -- Continuously running cron job that cancels itself to avoid running concurrently
@@ -84,6 +88,10 @@ WITH job AS (
         workflow.event != 'workflow_run' -- Filter out workflow_run-triggered jobs, which have nothing to do with the SHA
         AND workflow.event != 'repository_dispatch' -- Filter out repository_dispatch-triggered jobs, which have nothing to do with the SHA
         AND workflow.id in (select id from materialized_views.workflow_run_by_head_sha where head_sha = {sha: String})
+        AND (
+            {workflowId: Int64} = 0
+            OR workflow.id = {workflowId: Int64} -- If a specific workflow ID is provided, filter by it
+        )
         AND workflow.repository.full_name = {repo: String }
         AND workflow.name != 'Upload test stats while running' -- Continuously running cron job that cancels itself to avoid running concurrently
 )

torchci/components/commit/CommitStatus.tsx

@@ -54,10 +54,12 @@ function getBoxOrdering(jobs: JobData[], wideBoxes: Set<string>) {
 function WorkflowsContainer({
   jobs,
   unstableIssues,
+  workflowIdsByName,
   repoFullName,
 }: {
   jobs: JobData[];
   unstableIssues: IssueData[];
+  workflowIdsByName: Record<string, number[]>;
   repoFullName: string;
 }) {
   useScrollTo();
@@ -81,6 +83,7 @@ function WorkflowsContainer({
               repoFullName={repoFullName}
               key={workflowName}
               workflowName={workflowName}
+              allWorkflowIds={workflowIdsByName[workflowName] || []}
               jobs={jobs}
               unstableIssues={unstableIssues}
               wide={wideBoxes.has(workflowName)}
@@ -106,13 +109,15 @@ export default function CommitStatus({
   repoName,
   commit,
   jobs,
+  workflowIdsByName,
   isCommitPage,
   unstableIssues,
 }: {
   repoOwner: string;
   repoName: string;
   commit: CommitData;
   jobs: JobData[];
+  workflowIdsByName: Record<string, number[]>;
   isCommitPage: boolean;
   unstableIssues: IssueData[];
 }) {
@@ -174,6 +179,7 @@ export default function CommitStatus({
       />
       <WorkflowsContainer
         jobs={jobs}
+        workflowIdsByName={workflowIdsByName}
         unstableIssues={unstableIssues}
         repoFullName={`${repoOwner}/${repoName}`}
       />

torchci/components/commit/WorkflowBox.tsx

@@ -1,4 +1,4 @@
-import { Button, styled } from "@mui/material";
+import { Button, Stack, styled, Tooltip, Typography } from "@mui/material";
 import { TestInfo } from "components/additionalTestInfo/TestInfo";
 import styles from "components/commit/commit.module.css";
 import LogViewer, { SearchLogViewer } from "components/common/log/LogViewer";
@@ -14,7 +14,10 @@ import {
   ListUtilizationMetadataInfoAPIResponse,
   UtilizationMetadataInfo,
 } from "lib/utilization/types";
+import { CommitApiResponse } from "pages/api/[repoOwner]/[repoName]/commit/[sha]";
 import React, { useEffect, useState } from "react";
+import { FaInfoCircle } from "react-icons/fa";
+import useSWR from "swr";
 import useSWRImmutable from "swr/immutable";
 
 function sortJobsByConclusion(jobA: JobData, jobB: JobData): number {
@@ -140,16 +143,32 @@ export default function WorkflowBox({
   unstableIssues,
   wide,
   setWide,
+  allWorkflowIds,
   repoFullName,
 }: {
   workflowName: string;
   jobs: JobData[];
   unstableIssues: IssueData[];
   wide: boolean;
+  allWorkflowIds: number[];
   setWide: any;
   repoFullName: string;
 }) {
-  const workflowId = jobs[0].workflowId;
+  const [selectedWorkflowId, setSelectedWorkflowId] = useState<
+    string | undefined
+  >(undefined);
+  const workflowId = selectedWorkflowId || jobs[0].workflowId;
+
+  const { data: jobsFromSelectedWorkflowId } = useSWR<CommitApiResponse>(
+    selectedWorkflowId &&
+      `/api/${repoFullName}/commit/${jobs[0].sha}?workflowId=${selectedWorkflowId}`,
+    fetcher
+  );
+
+  if (selectedWorkflowId) {
+    jobs = jobsFromSelectedWorkflowId?.jobs || [];
+  }
+
   const isFailed = jobs.some(isFailedJob) !== false;
   const workflowClass = isFailed
     ? styles.workflowBoxFail
@@ -163,6 +182,7 @@ export default function WorkflowBox({
   const { artifacts, error } = useArtifacts(jobs.map((job) => job.workflowId));
   const [artifactsToShow, setArtifactsToShow] = useState(new Set<string>());
   const groupedArtifacts = groupArtifacts(jobs, artifacts);
+
   const [searchString, setSearchString] = useState("");
   const [searchRes, setSearchRes] = useState<{
     results: Map<string, LogSearchResult>;
@@ -184,11 +204,33 @@ export default function WorkflowBox({
     >
       <h3>{workflowName}</h3>
       <div>
-        <div
-          // Similar styling to an h4
-          style={{ float: "left", marginBottom: "1.33em", fontWeight: "bold" }}
-        >
-          Job Status
+        <div style={{ float: "left" }}>
+          <div
+            // Similar styling to an h4
+            style={{ marginBottom: "0.5em", fontWeight: "bold" }}
+          >
+            Job Status
+          </div>
+          <Stack direction="row" spacing={1}>
+            <select
+              value={selectedWorkflowId}
+              onChange={(e) => {
+                setSelectedWorkflowId(e.target.value);
+              }}
+            >
+              <option value={""}>Select Workflow ID</option>
+              {allWorkflowIds.sort().map((id) => (
+                <option key={id} value={id}>
+                  {id}
+                </option>
+              ))}
+            </select>
+            <Tooltip title="By default the box will show what it believes to be the latest jobs. Use this to select a specific workflow ID if it's wrong.">
+              <Typography>
+                <FaInfoCircle />
+              </Typography>
+            </Tooltip>
+          </Stack>
         </div>
         <div style={{ float: "right" }}>
           <div style={{ margin: ".5em 0em" }}>

torchci/lib/fetchCommit.ts

@@ -1,14 +1,21 @@
 import _ from "lodash";
 import { Octokit } from "octokit";
+import { CommitApiResponse } from "pages/api/[repoOwner]/[repoName]/commit/[sha]";
 import { queryClickhouseSaved } from "./clickhouse";
 import { commitDataFromResponse, getOctokit } from "./github";
 import { removeCancelledJobAfterRetry } from "./jobUtils";
-import { CommitData, JobData } from "./types";
+import { JobData } from "./types";
 
-async function fetchDatabaseInfo(owner: string, repo: string, sha: string) {
+async function fetchDatabaseInfo(
+  owner: string,
+  repo: string,
+  sha: string,
+  workflowId: number
+) {
   const response = await queryClickhouseSaved("commit_jobs_query", {
     repo: `${owner}/${repo}`,
     sha: sha,
+    workflowId,
   });
 
   for (const row of response) {
@@ -20,21 +27,53 @@ async function fetchDatabaseInfo(owner: string, repo: string, sha: string) {
   return response;
 }
 
+/**
+ * Get a mapping of workflow names to all workflow IDs from a list of job data.
+ * @param jobs
+ * @returns
+ */
+function getWorkflowIdsByName(jobs: JobData[]): Record<string, number[]> {
+  return _(jobs)
+    .groupBy((job) => job.workflowName)
+    .map((jobs, key) => {
+      const workflowIds = _(jobs)
+        .map((job) => job.workflowId)
+        .filter((id) => id !== null && id !== undefined)
+        .uniq()
+        .value();
+      return [key, workflowIds];
+    })
+    .fromPairs()
+    .value();
+}
+
+/**
+ *
+ * @param owner
+ * @param repo
+ * @param sha
+ * @param workflowId Optional workflow ID to filter jobs by. If not provided,
+ * all jobs for the commit will be returned.
+ * @returns
+ */
 export default async function fetchCommit(
   owner: string,
   repo: string,
-  sha: string
-): Promise<{ commit: CommitData; jobs: JobData[] }> {
+  sha: string,
+  workflowId: number = 0
+): Promise<CommitApiResponse> {
   // Retrieve commit data from GitHub
   const octokit = await getOctokit(owner, repo);
 
   const [githubResponse, response] = await Promise.all([
     octokit.rest.repos.getCommit({ owner, repo, ref: sha }),
-    await fetchDatabaseInfo(owner, repo, sha),
+    await fetchDatabaseInfo(owner, repo, sha, workflowId),
   ]);
 
   let jobs = response as any[];
 
+  const workflowIdsByName = getWorkflowIdsByName(jobs);
+
   // Subtle: we need to unique jobs by name, taking the most recent job. This is
   // because there might be many periodic jobs with the same name, and we want
   // to avoid noising up the display with many duplicate jobs.
@@ -65,6 +104,7 @@ export default async function fetchCommit(
   return {
     commit: commitDataFromResponse(githubResponse.data),
     jobs: _.concat(filteredJobs, badWorkflows),
+    workflowIdsByName,
   };
 }
 

torchci/pages/[repoOwner]/[repoName]/commit/[sha].tsx

@@ -43,7 +43,7 @@ export function CommitInfo({
     return <div>Loading...</div>;
   }
 
-  const { commit, jobs } = commitData;
+  const { commit, jobs, workflowIdsByName } = commitData;
 
   return (
     <div>
@@ -53,6 +53,7 @@ export function CommitInfo({
         repoName={repoName}
         commit={commit}
         jobs={jobs}
+        workflowIdsByName={workflowIdsByName}
         isCommitPage={true}
         unstableIssues={unstableIssuesData ?? []}
       />

torchci/pages/[repoOwner]/[repoName]/pull/[prNumber].tsx

@@ -48,14 +48,15 @@ function CommitInfo({
   if (commitData === undefined) {
     return <div>Loading...</div>;
   }
-  const { commit, jobs } = commitData;
+  const { commit, jobs, workflowIdsByName } = commitData;
 
   return (
     <CommitStatus
       repoOwner={repoOwner}
       repoName={repoName}
       commit={commit}
       jobs={jobs}
+      workflowIdsByName={workflowIdsByName}
       isCommitPage={false}
       unstableIssues={unstableIssuesData ?? []}
     />

torchci/pages/api/[repoOwner]/[repoName]/commit/[sha].ts

@@ -5,19 +5,22 @@ import type { NextApiRequest, NextApiResponse } from "next";
 export type CommitApiResponse = {
   commit: CommitData;
   jobs: JobData[];
+  workflowIdsByName: Record<string, number[]>;
 };
 
 export default async function handler(
   req: NextApiRequest,
   res: NextApiResponse<CommitApiResponse>
 ) {
+  const workflowId = parseInt(req.query.workflowId as string, 10) || 0;
   res
     .status(200)
     .json(
       await fetchCommit(
         req.query.repoOwner as string,
         req.query.repoName as string,
-        req.query.sha as string
+        req.query.sha as string,
+        workflowId
       )
     );
 }