Optimize config unroll with parallel processing for blocks and sections

packages/config-yaml/src/load/unroll.ts

@@ -24,7 +24,7 @@ import {
   packageIdentifierToShorthandSlug,
   useProxyForUnrenderedSecrets,
 } from "./clientRender.js";
-import { getBlockType } from "./getBlockType.js";
+import { BlockType, getBlockType } from "./getBlockType.js";
 
 export function parseConfigYaml(configYaml: string): ConfigYaml {
   try {
@@ -322,12 +322,15 @@ export async function unrollBlocks(
     "name" | "version" | "rules" | "schema" | "metadata"
   >)[] = ["models", "context", "data", "mcpServers", "prompts", "docs"];
 
-  // For each section, replace "uses/with" blocks with the real thing
-  for (const section of sections) {
-    if (assistant[section]) {
-      const sectionBlocks: any[] = [];
+  // Process all sections in parallel
+  const sectionPromises = sections.map(async (section) => {
+    if (!assistant[section]) {
+      return { section, blocks: null };
+    }
 
-      for (const unrolledBlock of assistant[section]) {
+    // Process all blocks in this section in parallel
+    const blockPromises = assistant[section].map(
+      async (unrolledBlock, index) => {
         // "uses/with" block
         if ("uses" in unrolledBlock) {
           try {
@@ -338,10 +341,13 @@ export async function unrollBlocks(
             );
             const block = blockConfigYaml[section]?.[0];
             if (block) {
-              sectionBlocks.push(
-                mergeOverrides(block, unrolledBlock.override ?? {}),
-              );
+              return {
+                index,
+                block: mergeOverrides(block, unrolledBlock.override ?? {}),
+                error: null,
+              };
             }
+            return { index, block: null, error: null };
           } catch (err) {
             let msg = "";
             if (
@@ -353,99 +359,190 @@ export async function unrollBlocks(
               msg = `${(err as Error).message}.\n> ${JSON.stringify(unrolledBlock.uses)}`;
             }
 
-            errors.push({
-              fatal: false,
-              message: msg,
-            });
-
             console.error(
               `Failed to unroll block ${JSON.stringify(unrolledBlock.uses)}: ${(err as Error).message}`,
             );
-            sectionBlocks.push(null);
+
+            return {
+              index,
+              block: null,
+              error: {
+                fatal: false,
+                message: msg,
+              },
+            };
           }
         } else {
           // Normal block
-          sectionBlocks.push(unrolledBlock);
+          return { index, block: unrolledBlock, error: null };
         }
-      }
+      },
+    );
 
-      unrolledAssistant[section] = sectionBlocks;
+    const blockResults = await Promise.all(blockPromises);
+
+    // Collect errors and maintain order
+    const sectionBlocks: any[] = [];
+    const sectionErrors: ConfigValidationError[] = [];
+
+    for (const result of blockResults) {
+      if (result.error) {
+        sectionErrors.push(result.error);
+      }
+      sectionBlocks[result.index] = result.block;
     }
-  }
 
-  // Rules are a bit different because they can be strings, so handle separately
-  if (assistant.rules) {
-    const rules: (Rule | null)[] = [];
-    for (const rule of assistant.rules) {
-      if (typeof rule === "string" || !("uses" in rule)) {
-        rules.push(rule);
-      } else if ("uses" in rule) {
-        try {
-          const blockConfigYaml = await resolveBlock(
-            decodePackageIdentifier(rule.uses),
-            rule.with,
-            registry,
-          );
-          const block = blockConfigYaml.rules?.[0];
-          if (block) {
-            rules.push(block);
+    return { section, blocks: sectionBlocks, errors: sectionErrors };
+  });
+
+  // Process rules in parallel
+  const rulesPromise = assistant.rules
+    ? (async () => {
+        const rulePromises = assistant.rules!.map(async (rule, index) => {
+          if (typeof rule === "string" || !("uses" in rule)) {
+            return { index, rule, error: null };
+          } else if ("uses" in rule) {
+            try {
+              const blockConfigYaml = await resolveBlock(
+                decodePackageIdentifier(rule.uses),
+                rule.with,
+                registry,
+              );
+              const block = blockConfigYaml.rules?.[0];
+              return { index, rule: block || null, error: null };
+            } catch (err) {
+              console.error(
+                `Failed to unroll block ${rule.uses}: ${(err as Error).message}`,
+              );
+
+              return {
+                index,
+                rule: null,
+                error: {
+                  fatal: false,
+                  message: `${(err as Error).message}:\n${rule.uses}`,
+                },
+              };
+            }
+          }
+          return { index, rule: null, error: null };
+        });
+
+        const ruleResults = await Promise.all(rulePromises);
+
+        const rules: (Rule | null)[] = [];
+        const ruleErrors: ConfigValidationError[] = [];
+
+        for (const result of ruleResults) {
+          if (result.error) {
+            ruleErrors.push(result.error);
           }
-        } catch (err) {
-          errors.push({
-            fatal: false,
-            message: `${(err as Error).message}:\n${rule.uses}`,
-          });
-
-          console.error(
-            `Failed to unroll block ${rule.uses}: ${(err as Error).message}`,
-          );
-          rules.push(null);
+          rules[result.index] = result.rule;
         }
-      }
-    }
 
-    unrolledAssistant.rules = rules;
-  }
+        return { rules, errors: ruleErrors };
+      })()
+    : Promise.resolve({ rules: undefined, errors: [] });
 
-  // Add injected blocks
-  for (const injectBlock of injectBlocks ?? []) {
-    try {
-      const blockConfigYaml = await registry.getContent(injectBlock);
-      const parsedBlock = parseMarkdownRuleOrConfigYaml(
-        blockConfigYaml,
-        injectBlock,
-      );
-      const blockType = getBlockType(parsedBlock);
-      const resolvedBlock = await resolveBlock(
-        injectBlock,
-        undefined,
-        registry,
-      );
+  // Process injected blocks in parallel
+  const injectedBlocksPromise = injectBlocks
+    ? (async () => {
+        const injectedBlockPromises = injectBlocks.map(async (injectBlock) => {
+          try {
+            const blockConfigYaml = await registry.getContent(injectBlock);
+            const parsedBlock = parseMarkdownRuleOrConfigYaml(
+              blockConfigYaml,
+              injectBlock,
+            );
+            const blockType = getBlockType(parsedBlock);
+            const resolvedBlock = await resolveBlock(
+              injectBlock,
+              undefined,
+              registry,
+            );
+
+            return {
+              blockType,
+              resolvedBlock,
+              error: null,
+            };
+          } catch (err) {
+            let msg = "";
+            if (injectBlock.uriType === "file") {
+              msg = `${(err as Error).message}.\n> ${injectBlock.filePath}`;
+            } else {
+              msg = `${(err as Error).message}.\n> ${injectBlock.fullSlug}`;
+            }
 
-      if (blockType) {
-        if (!unrolledAssistant[blockType]) {
-          unrolledAssistant[blockType] = [];
+            console.error(
+              `Failed to unroll block ${JSON.stringify(injectBlock)}: ${(err as Error).message}`,
+            );
+
+            return {
+              blockType: null,
+              resolvedBlock: null,
+              error: {
+                fatal: false,
+                message: msg,
+              },
+            };
+          }
+        });
+
+        const injectedResults = await Promise.all(injectedBlockPromises);
+        const injectedErrors: ConfigValidationError[] = [];
+        const injectedBlocks: { blockType: string; resolvedBlock: any }[] = [];
+
+        for (const result of injectedResults) {
+          if (result.error) {
+            injectedErrors.push(result.error);
+          } else if (result.blockType && result.resolvedBlock) {
+            injectedBlocks.push({
+              blockType: result.blockType,
+              resolvedBlock: result.resolvedBlock,
+            });
+          }
         }
-        unrolledAssistant[blockType]?.push(
-          ...(resolvedBlock[blockType] as any),
-        );
-      }
-    } catch (err) {
-      let msg = "";
-      if (injectBlock.uriType === "file") {
-        msg = `${(err as Error).message}.\n> ${injectBlock.filePath}`;
-      } else {
-        msg = `${(err as Error).message}.\n> ${injectBlock.fullSlug}`;
-      }
-      errors.push({
-        fatal: false,
-        message: msg,
-      });
 
-      console.error(
-        `Failed to unroll block ${JSON.stringify(injectBlock)}: ${(err as Error).message}`,
-      );
+        return { injectedBlocks, errors: injectedErrors };
+      })()
+    : Promise.resolve({ injectedBlocks: [], errors: [] });
+
+  // Wait for all parallel operations to complete
+  const [sectionResults, rulesResult, injectedResult] = await Promise.all([
+    Promise.all(sectionPromises),
+    rulesPromise,
+    injectedBlocksPromise,
+  ]);
+
+  // Collect all errors
+  for (const sectionResult of sectionResults) {
+    if (sectionResult.errors) {
+      errors.push(...sectionResult.errors);
+    }
+  }
+  errors.push(...rulesResult.errors);
+  errors.push(...injectedResult.errors);
+
+  // Assign section results
+  for (const sectionResult of sectionResults) {
+    if (sectionResult.blocks) {
+      unrolledAssistant[sectionResult.section] = sectionResult.blocks;
+    }
+  }
+
+  // Assign rules result
+  if (rulesResult.rules) {
+    unrolledAssistant.rules = rulesResult.rules;
+  }
+
+  // Add injected blocks
+  for (const { blockType, resolvedBlock } of injectedResult.injectedBlocks) {
+    const key = blockType as BlockType;
+    if (!unrolledAssistant[key]) {
+      unrolledAssistant[key] = [];
     }
+    unrolledAssistant[key]?.push(...(resolvedBlock[blockType] as any));
   }
 
   const configResult: ConfigResult<AssistantUnrolled> = {