Improve failed build handling

bun.lock

@@ -14,7 +14,7 @@
     },
     "clients/typescript": {
       "name": "@metorial-services/forge-client",
-      "version": "1.0.6",
+      "version": "1.0.0",
       "dependencies": {
         "@lowerdeck/rpc-client": "^1.0.2",
       },
@@ -33,6 +33,7 @@
         "@aws-sdk/client-iam": "^3.957.0",
         "@lowerdeck/api-mux": "^1.0.1",
         "@lowerdeck/cron": "^1.0.4",
+        "@lowerdeck/delay": "^1.0.4",
         "@lowerdeck/encryption": "^1.0.6",
         "@lowerdeck/env": "^1.0.4",
         "@lowerdeck/error": "^1.0.8",

service/package.json

@@ -19,6 +19,7 @@
     "@aws-sdk/client-iam": "^3.957.0",
     "@lowerdeck/api-mux": "^1.0.1",
     "@lowerdeck/cron": "^1.0.4",
+    "@lowerdeck/delay": "^1.0.4",
     "@lowerdeck/encryption": "^1.0.6",
     "@lowerdeck/env": "^1.0.4",
     "@lowerdeck/error": "^1.0.8",

service/src/providers/aws-codebuild/index.ts

@@ -73,7 +73,6 @@ let startBuildQueueProcessor = startAwsCodeBuildQueue.process(async data => {
       environmentVariablesOverride: [
         ...Object.entries({
           ...envVars,
-
           WORKFLOW_RUN_ID: ctx.run.id,
           WORKFLOW_VERSION_ID: version.id,
           RUNTIME: '[email protected]'
@@ -492,20 +491,17 @@ let buildEndedQueueProcessor = buildEndedQueue.process(async data => {
 
   let ctx = await BuildContext.of(data.runId);
 
+  let failed = build.buildStatus != 'SUCCEEDED';
+
   await db.workflowRunStep.updateMany({
     where: { runOid: ctx.run.oid, status: 'running' },
-    data: { status: 'failed', endedAt: new Date() }
+    data: { status: failed ? 'failed' : 'succeeded', endedAt: new Date() }
   });
   await db.workflowRunStep.updateMany({
     where: { runOid: ctx.run.oid, status: 'pending' },
     data: { status: 'canceled' }
   });
 
-  let hasFailedSteps = (await db.workflowRunStep.findFirst({
-    where: { runOid: ctx.run.oid, status: 'failed' }
-  }))!!;
-  let failed = hasFailedSteps || build.buildStatus != 'SUCCEEDED';
-
   await db.workflowRun.updateMany({
     where: { id: data.runId },
     data: {

service/src/services/workflowVersion.ts

@@ -38,7 +38,7 @@ class workflowVersionServiceImpl {
       steps: WorkflowVersionSteps[];
     };
   }) {
-    let identifier = generatePlainId(6);
+    let identifier = generatePlainId(12);
 
     return await db.$transaction(async db => {
       let version = await db.workflowVersion.create({

service/src/storage.ts

@@ -1,9 +1,23 @@
+import { delay } from '@lowerdeck/delay';
 import { ObjectStorageClient } from 'object-storage-client';
 import { env } from './env';
 
 export let storage = new ObjectStorageClient(env.storage.OBJECT_STORAGE_URL);
 
-setTimeout(async () => {
+let initBuckets = async () => {
   await storage.upsertBucket(env.storage.LOG_BUCKET_NAME);
   await storage.upsertBucket(env.storage.ARTIFACT_BUCKET_NAME);
-}, 5000);
+};
+
+async () => {
+  while (true) {
+    try {
+      await initBuckets();
+      return;
+    } catch (err) {
+      console.error('Error initializing storage buckets, retrying...');
+    }
+
+    await delay(5000);
+  }
+};