Enable build job retry

Enable job retry service to trigger retry
for failed/incomplete build jobs.

Signed-off-by: Jeny Sadadia <[email protected]>

Author: Jeny Sadadia

src/job_retry.py

@@ -21,9 +21,7 @@ def _setup(self, args):
         return self._api_helper.subscribe_filters({
             "state": "done",
             "result": ("fail", "incomplete"),
-            "kind": "job"
-            # ToDo: Retry for build jobs
-            # "kind": ("kbuild", "job")
+            "kind": ("kbuild", "job"),
         })
 
     def _stop(self, sub_id):
@@ -63,17 +61,21 @@ def _run(self, sub_id):
                 continue
 
             parent_kind = None
-            if node.get('kind') == 'job':
-                parent_kind = 'kbuild'
-            # ToDo: retry build jobs
-            # if node.get("kind") == "kbuild":
-            #     parent_kind = "checkout"
+            if node.get("kind") == "job":
+                parent_kind = "kbuild"
+            if node.get("kind") == "kbuild":
+                parent_kind = "checkout"
             if parent_kind:
                 event_data = self._find_parent_kind(node, self._api_helper, parent_kind)
                 if not event_data:
                     self.log.error(f"Not able to find parent node for {node['id']}")
                     continue
-                event_data["jobfilter"] = [node["name"]]
+                if node["kind"] == "kbuild":
+                    event_data["jobfilter"] = [f'{node["name"]}+']
+                else:
+                    event_data["jobfilter"] = [node["name"]]
+                # Change event data state to available to trigger jobs based on scheduler configs
+                event_data["state"] = "available"
                 event_data["platform_filter"] = [node["data"].get("platform")]
                 event_data["retry_counter"] = retry_counter + 1
                 event_data["debug"] = {"retry_by": str(node["id"])}

src/scheduler.py

@@ -411,6 +411,7 @@ def _run_scheduler(self, channel, sub_id):
                 if not event:
                     # If we received a keep-alive event, just continue
                     continue
+                self.log.debug(f"Event received: {channel}-{event}")
             except Exception as e:
                 with self._stop_thread_lock:
                     if self._stop_thread:
@@ -427,8 +428,11 @@ def _run_scheduler(self, channel, sub_id):
                 continue
             subscribe_retries = 0
             for job, runtime, platform, rules in self._sched.get_schedule(event):
+                self.log.debug(f"---Runtime: {runtime.config.name}")
                 input_node = self._api.node.get(event['id'])
+                self.log.debug(f"Checking input_node: {input_node['id']}")
                 jobfilter = event.get('jobfilter')
+                self.log.debug(f"Job filter: {jobfilter}-{input_node['id']}")
                 # Add to node data the jobfilter if it exists in event
                 if jobfilter and isinstance(jobfilter, list):
                     input_node['jobfilter'] = jobfilter
@@ -441,8 +445,10 @@ def _run_scheduler(self, channel, sub_id):
                         continue
                 if not self._verify_architecture_filter(job, input_node):
                     continue
+                self.log.debug(f"Checking if node can be created: {input_node['id']}")
                 with self._api_helper_lock:
                     flag = self._api_helper.should_create_node(rules, input_node)
+                    self.log.debug(f"should create node flag: {flag} {input_node['id']}")
                 if flag:
                     retry_counter = event.get('retry_counter', 0)
                     self._run_job(job, runtime, platform, input_node, retry_counter)