Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions cloud/packages/ci-manager/src/build-store.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { BuildInfo, BuildEvent, Status } from "./types";
import { randomUUID } from "crypto";
import { dirname, join } from "path";
import { mkdir, rm } from "fs/promises";
import { join, dirname } from "path";
import { createNanoEvents } from "nanoevents";
import type { BuildEvent, BuildInfo, Status } from "./types";

export class BuildStore {
private builds = new Map<string, BuildInfo>();
Expand All @@ -12,7 +12,7 @@ export class BuildStore {
"status-change": (buildId: string, status: Status) => void;
}>();

constructor(tempDir: string = "/tmp/ci-builds") {
constructor(tempDir = "/tmp/ci-builds") {
this.tempDir = tempDir;
}

Expand All @@ -25,7 +25,7 @@ export class BuildStore {
dockerfilePath: string,
environmentId: string,
buildArgs: Record<string, string>,
buildTarget: string | undefined
buildTarget: string | undefined,
): string {
const id = randomUUID();
const contextPath = join(this.tempDir, id, "context.tar.gz");
Expand Down Expand Up @@ -94,7 +94,7 @@ export class BuildStore {
}
}

getContextPath(id: string): string | undefined{
getContextPath(id: string): string | undefined {
return this.builds.get(id)?.contextPath;
}

Expand Down
10 changes: 4 additions & 6 deletions cloud/packages/ci-manager/src/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@ interface KanikoArguments {
}

// SAFETY: buildArgs keys never have equal signs or spaces
function convertBuildArgsToArgs(
buildArgs: Record<string, string>,
): string[] {
function convertBuildArgsToArgs(buildArgs: Record<string, string>): string[] {
return Object.entries(buildArgs).flatMap(([key, value]) => [
`--build-arg`,
`${key}=${value}`,
Expand All @@ -34,11 +32,11 @@ export function serializeKanikoArguments(args: KanikoArguments): string {
"--no-push",
"--single-snapshot",
"--verbosity=info",
].map(arg => {
].map((arg) => {
// Args should never contain UNIT_SEP_CHAR, but we can
// escape it if they do.
return arg.replaceAll(UNIT_SEP_CHAR, "\\" + UNIT_SEP_CHAR)
return arg.replaceAll(UNIT_SEP_CHAR, "\\" + UNIT_SEP_CHAR);
});

return preparedArgs.join(UNIT_SEP_CHAR);
}
}
38 changes: 20 additions & 18 deletions cloud/packages/ci-manager/src/executors/docker.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { spawn } from "node:child_process";
import { BuildStore } from "../build-store";
import { serializeKanikoArguments, UNIT_SEP_CHAR } from "../common";
import type { BuildStore } from "../build-store";
import { serializeKanikoArguments } from "../common";

export async function runDockerBuild(
buildStore: BuildStore,
Expand All @@ -20,16 +20,14 @@ export async function runDockerBuild(
"--rm",
"--network=host",
"-e",
`KANIKO_ARGS=${
serializeKanikoArguments({
contextUrl,
outputUrl,
destination: `${buildId}:latest`,
dockerfilePath: build.dockerfilePath,
buildArgs: build.buildArgs,
buildTarget: build.buildTarget,
})
}`,
`KANIKO_ARGS=${serializeKanikoArguments({
contextUrl,
outputUrl,
destination: `${buildId}:latest`,
dockerfilePath: build.dockerfilePath,
buildArgs: build.buildArgs,
buildTarget: build.buildTarget,
})}`,
"ci-runner",
];

Expand All @@ -40,12 +38,12 @@ export async function runDockerBuild(

buildStore.updateStatus(buildId, {
type: "running",
data: { docker: {} }
data: { docker: {} },
});

return new Promise<void>((resolve, reject) => {
const dockerProcess = spawn("docker", kanikoArgs, {
stdio: ["pipe", "pipe", "pipe"]
stdio: ["pipe", "pipe", "pipe"],
});

buildStore.setContainerProcess(buildId, dockerProcess);
Expand All @@ -71,7 +69,10 @@ export async function runDockerBuild(
});

dockerProcess.on("close", (code) => {
buildStore.addLog(buildId, `Docker process closed with exit code: ${code}`);
buildStore.addLog(
buildId,
`Docker process closed with exit code: ${code}`,
);
buildStore.updateStatus(buildId, { type: "finishing", data: {} });

if (code === 0) {
Expand All @@ -90,7 +91,10 @@ export async function runDockerBuild(
});

dockerProcess.on("error", (error) => {
buildStore.addLog(buildId, `Docker process error: ${error.message}`);
buildStore.addLog(
buildId,
`Docker process error: ${error.message}`,
);
buildStore.updateStatus(buildId, {
type: "failure",
data: { reason: `Failed to start kaniko: ${error.message}` },
Expand All @@ -99,5 +103,3 @@ export async function runDockerBuild(
});
});
}


8 changes: 4 additions & 4 deletions cloud/packages/ci-manager/src/executors/rivet.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { RivetClient } from "@rivet-gg/api";
import { BuildStore } from "../build-store";
import type { BuildStore } from "../build-store";
import { serializeKanikoArguments } from "../common";

export async function runRivetBuild(
Expand Down Expand Up @@ -57,7 +57,7 @@ export async function runRivetBuild(
dockerfilePath: build.dockerfilePath,
buildArgs: build.buildArgs,
buildTarget: build.buildTarget,
})
}),
},
},
network: {
Expand All @@ -81,8 +81,8 @@ export async function runRivetBuild(
buildStore.updateStatus(buildId, {
type: "running",
data: {
rivet: { actorId }
}
rivet: { actorId },
},
});

await pollActorStatus(
Expand Down
8 changes: 4 additions & 4 deletions cloud/packages/ci-manager/src/kaniko-runner.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { BuildStore } from "./build-store";
import { mkdir } from "node:fs/promises";
import { existsSync } from "node:fs";
import { mkdir } from "node:fs/promises";
import { dirname } from "node:path";
import type { BuildStore } from "./build-store";
import { runDockerBuild } from "./executors/docker";
import { runRivetBuild } from "./executors/rivet";

Expand All @@ -20,8 +20,8 @@ export async function runKanikoBuild(
buildStore.updateStatus(buildId, {
type: "running",
data: {
noRunner: {}
}
noRunner: {},
},
});

const executionMode = process.env.KANIKO_EXECUTION_MODE || "docker";
Expand Down
59 changes: 37 additions & 22 deletions cloud/packages/ci-manager/src/oci-converter.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import { execSync } from "child_process";
import { mkdir, rm, writeFile, readFile, readdir, stat } from "fs/promises";
import { join, dirname } from "path";
import { createReadStream, createWriteStream } from "fs";
import { pipeline } from "stream/promises";
import { join } from "path";
import { mkdir, readFile, rm, writeFile } from "fs/promises";
import * as tar from "tar";

export interface OCIConversionResult {
Expand All @@ -12,14 +10,14 @@ export interface OCIConversionResult {

export async function convertDockerTarToOCIBundle(
dockerTarPath: string,
tempDir: string = "/tmp/oci-conversion"
tempDir = "/tmp/oci-conversion",
): Promise<OCIConversionResult> {
const conversionId = Math.random().toString(36).substring(7);
const workDir = join(tempDir, conversionId);

try {
await mkdir(workDir, { recursive: true });

const dockerImagePath = join(workDir, "docker-image.tar");
const ociImagePath = join(workDir, "oci-image");
const ociBundlePath = join(workDir, "oci-bundle");
Expand All @@ -30,54 +28,71 @@ export async function convertDockerTarToOCIBundle(
await writeFile(dockerImagePath, dockerTarData);

// Convert Docker image to OCI image using skopeo
console.log(`Converting Docker image to OCI image: ${dockerImagePath} -> ${ociImagePath}`);
execSync(`skopeo copy docker-archive:${dockerImagePath} oci:${ociImagePath}:default`, {
stdio: "pipe"
});
console.log(
`Converting Docker image to OCI image: ${dockerImagePath} -> ${ociImagePath}`,
);
execSync(
`skopeo copy docker-archive:${dockerImagePath} oci:${ociImagePath}:default`,
{
stdio: "pipe",
},
);

// Convert OCI image to OCI bundle using umoci
console.log(`Converting OCI image to OCI bundle: ${ociImagePath} -> ${ociBundlePath}`);
execSync(`umoci unpack --rootless --image ${ociImagePath}:default ${ociBundlePath}`, {
stdio: "pipe"
});
console.log(
`Converting OCI image to OCI bundle: ${ociImagePath} -> ${ociBundlePath}`,
);
execSync(
`umoci unpack --rootless --image ${ociImagePath}:default ${ociBundlePath}`,
{
stdio: "pipe",
},
);

// Create tar from OCI bundle
console.log(`Creating tar from OCI bundle: ${ociBundlePath} -> ${bundleTarPath}`);
console.log(
`Creating tar from OCI bundle: ${ociBundlePath} -> ${bundleTarPath}`,
);
await tar.create(
{
file: bundleTarPath,
cwd: ociBundlePath,
},
["."]
["."],
);

// Clean up intermediate files
await Promise.all([
rm(dockerImagePath, { force: true }),
rm(ociImagePath, { recursive: true, force: true }),
rm(ociBundlePath, { recursive: true, force: true })
rm(ociBundlePath, { recursive: true, force: true }),
]);

const cleanup = async () => {
try {
await rm(workDir, { recursive: true, force: true });
} catch (error) {
console.warn(`Failed to cleanup OCI conversion directory ${workDir}:`, error);
console.warn(
`Failed to cleanup OCI conversion directory ${workDir}:`,
error,
);
}
};

return {
bundleTarPath,
cleanup
cleanup,
};
} catch (error) {
// Cleanup on error
try {
await rm(workDir, { recursive: true, force: true });
} catch (cleanupError) {
console.warn(`Failed to cleanup after error in ${workDir}:`, cleanupError);
console.warn(
`Failed to cleanup after error in ${workDir}:`,
cleanupError,
);
}
throw new Error(`OCI conversion failed: ${error}`);
}
}

7 changes: 2 additions & 5 deletions cloud/packages/ci-manager/src/rivet-uploader.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import { statSync } from "fs";
import { RivetClient } from "@rivet-gg/api";
import { warn } from "console";
import { createReadStream, statSync } from "fs";
import { readFile } from "fs/promises";

export interface RivetUploadConfig {
token: string;
Expand Down Expand Up @@ -113,7 +111,7 @@ export async function uploadOCIBundleToRivet(
async function uploadChunkWithRetry(
url: string,
buffer: Buffer,
maxRetries: number = 3,
maxRetries = 3,
): Promise<void> {
let lastError: Error | null = null;

Expand Down Expand Up @@ -232,4 +230,3 @@ async function patchBuildTags(
// Don't throw here to avoid failing the entire upload process
}
}

Loading
Loading