Use Vulkan GPU inside container to run AI models

Signed-off-by: Ansuman Sahoo <[email protected]>

Author: unsuman

pkg/driver/krunkit/hack/install-vulkan-gpu.sh

@@ -5,7 +5,6 @@ package krunkit
 
 import (
 	"context"
-	_ "embed"
 	"errors"
 	"fmt"
 	"net"
@@ -39,25 +38,9 @@ type LimaKrunkitDriver struct {
 	krunkitWaitCh chan error
 }
 
-type KrunkitOpts struct {
-	GPUAccel *bool `yaml:"gpuAccel,omitempty"`
-}
-
-func NewKrunkitOpts(cfg *limatype.LimaYAML) (*KrunkitOpts, error) {
-	var krunkitOpts KrunkitOpts
-	if err := limayaml.Convert(cfg.VMOpts[vmType], &krunkitOpts, "vmOpts.krunkit"); err != nil {
-		return nil, err
-	}
-
-	return &krunkitOpts, nil
-}
-
 var (
 	_      driver.Driver   = (*LimaKrunkitDriver)(nil)
 	vmType limatype.VMType = "krunkit"
-
-	//go:embed hack/install-vulkan-gpu.sh
-	gpuProvisionScript string
 )
 
 func New() *LimaKrunkitDriver {
@@ -188,17 +171,6 @@ func validateConfig(cfg *limatype.LimaYAML) error {
 		return fmt.Errorf("field `mountType` must be %q or %q for krunkit driver, got %q", limatype.VIRTIOFS, limatype.REVSSHFS, *cfg.MountType)
 	}
 
-	// If GPU acceleration is requested, ensure Fedora image/template is used
-	krunkitOpts, err := NewKrunkitOpts(cfg)
-	if err != nil {
-		return err
-	}
-	if krunkitOpts.GPUAccel != nil && *krunkitOpts.GPUAccel {
-		if !isFedoraConfigured(cfg) {
-			logrus.Warn("gpuAccel: true requires a Fedora image (use a Fedora base template or image)")
-		}
-	}
-
 	return nil
 }
 
@@ -231,52 +203,20 @@ func (l *LimaKrunkitDriver) FillConfig(_ context.Context, cfg *limatype.LimaYAML
 
 	cfg.VMType = ptr.Of(vmType)
 
-	krunkitOpts, err := NewKrunkitOpts(cfg)
-	if err != nil {
-		return err
-	}
-
-	if krunkitOpts.GPUAccel == nil {
-		krunkitOpts.GPUAccel = ptr.Of(false)
-	}
-
-	if *krunkitOpts.GPUAccel {
-		gpuInstallScript := limatype.Provision{
-			Mode:   limatype.ProvisionModeData,
-			Script: ptr.Of(gpuProvisionScript),
-			ProvisionData: limatype.ProvisionData{
-				Content:     ptr.Of(gpuProvisionScript),
-				Path:        ptr.Of("/usr/local/bin/install-vulkan-gpu.sh"),
-				Permissions: ptr.Of("0755"),
-				Overwrite:   ptr.Of(false),
-				Owner:       cfg.User.Name,
-			},
-		}
-		cfg.Provision = append(cfg.Provision, gpuInstallScript)
-		cfg.Message = "To enable GPU support for krunkit, run the following command inside the VM:\n\033[32msudo install-vulkan-gpu.sh\033[0m\n"
-	}
-
 	return validateConfig(cfg)
 }
 
 func (l *LimaKrunkitDriver) BootScripts() (map[string][]byte, error) {
-	// Override default reboot-if-required with a no-op because Fedora does not support this well and
-	// takes a long time to start up.
-	krunkitOpts, err := NewKrunkitOpts(l.Instance.Config)
-	if err != nil {
-		return nil, err
-	}
-	if krunkitOpts.GPUAccel == nil || !*krunkitOpts.GPUAccel {
-		return nil, nil
-	}
-	scripts := map[string][]byte{
-		"00-reboot-if-required.sh": []byte(`#!/bin/sh
+	//Disabled by krunkit driver for Fedora to make boot time faster
+	if isFedoraConfigured(l.Instance.Config) {
+		return map[string][]byte{
+			"00-reboot-if-required.sh": []byte(`#!/bin/sh
 set -eu
-# Disabled by krunkit driver
 exit 0
 `),
+		}, nil
 	}
-	return scripts, nil
+	return nil, nil
 }
 
 func (l *LimaKrunkitDriver) Create(_ context.Context) error {

pkg/driver/krunkit/krunkit_driver_darwin_arm64.go

@@ -31,59 +31,52 @@ go build -o <PREFIX>/libexec/lima/lima-driver-krunkit ./cmd/lima-driver-krunkit/
 limactl info   # "vmTypes" should include "krunkit"
 ```
 
-
 ## Quick start
 
-- Non‑GPU (general workloads)
-```bash
-limactl start default --vm-type=krunkit
-```
-
-- GPU (Vulkan via Venus)
-  - Recommended distro: Fedora 40+ (smoothest Mesa/Vulkan setup; uses COPR “slp/mesa-krunkit” for patched mesa-vulkan-drivers).
-  - Start from the krunkit template and follow the logs to complete GPU setup.
+Start a krunkit VM with rootful containerd (required for granting containers access to `/dev/dri` and the Vulkan API):
 
 {{< tabpane text=true >}}
 {{% tab header="CLI" %}}
 ```bash
-# GPU (Vulkan via Venus on Fedora)
-limactl start template:experimental/krunkit
+limactl start default --vm-type=krunkit --containerd=system
+limactl shell default
 ```
 {{% /tab %}}
 {{% tab header="YAML" %}}
 ```yaml
 vmType: krunkit
 
-# For AI workloads, at least 4GiB memory and 4 CPUs are recommended.
-memory: 4GiB
-cpus: 4
-arch: aarch64
-
-# Fedora 40+ is preferred for Mesa & Vulkan (Venus) support
 base:
-- template://_images/fedora
-
-mounts:
-- location: "~"
-  writable: true
-
-mountType: virtiofs
+- template://_images/ubuntu
+- template://_default/mounts
 
-vmOpts:
-  krunkit:
-    gpuAccel: true
+containerd:
+  system: true
 ```
 {{% /tab %}}
 {{< /tabpane >}}
 
-After the VM is READY, inside the VM:
+Run AI models with [`llama.cpp`](https://github.com/ggml-org/llama.cpp) in a container using `nerdctl`:
+
+1) Place a `.gguf` model inside the VM, e.g. `~/models/YourModel.gguf` or download it from [Hugging Face](https://huggingface.co/models?library=gguf)
+
+2) Launch the container with GPU device nodes and bind‑mount the model directory:
+
+```bash
+sudo nerdctl run --rm -ti \
+  --device /dev/dri \
+  -v ~/models:/models \
+  quay.io/slopezpa/fedora-vgpu-llama
+```
+
+3) Inside the container, run llama.cpp:
+
 ```bash
-sudo install-vulkan-gpu.sh
+llama-cli -m /models/YourModel.gguf -b 512 -ngl 99 -p "Introduce yourself"
 ```
 
 ## Notes and caveats
 - macOS Ventura or later on Apple Silicon is required.
-- GPU mode requires a Fedora image/template; Fedora 40+ recommended for Mesa/Vulkan (Venus).
-- To verify GPU/Vulkan in the guest, use tools like `vulkaninfo` after running the install script.
-- `Libkrun` and [`Ramalama`](https://github.com/containers/ramalama)(a tool that simplifies running AI models locally) use CPU inferencing as of **July 2, 2025** and are actively working to support GPU inferencing. [More info](https://developers.redhat.com/articles/2025/07/02/supercharging-ai-isolation-microvms-ramalama-libkrun#current_limitations_and_future_directions__gpu_enablement).
+- Rootful containerd (`--containerd=system`) is necessary to pass through /dev/dri to containers.
+- To verify GPU/Vulkan in the guest container, use tools like `vulkaninfo`.
 - Driver architecture details: see [Virtual Machine Drivers](../../dev/drivers).