From 3e38c06ad7fe3f9cc3cb2092a12b273586a712cd Mon Sep 17 00:00:00 2001 From: Joey Spiker Date: Sat, 19 Jul 2025 10:20:41 -0400 Subject: [PATCH 1/3] fix for running llamasharp inside containers based on nvidia/cuda and detecting version --- LLama/Native/Load/SystemInfo.cs | 35 ++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/LLama/Native/Load/SystemInfo.cs b/LLama/Native/Load/SystemInfo.cs index f2e6827ec..9e6d3e4fa 100644 --- a/LLama/Native/Load/SystemInfo.cs +++ b/LLama/Native/Load/SystemInfo.cs @@ -41,19 +41,19 @@ public static SystemInfo Get() return new SystemInfo(platform, GetCudaMajorVersion(), GetVulkanVersion()); } - + #region Vulkan version private static string? GetVulkanVersion() { // Get Vulkan Summary string? vulkanSummary = GetVulkanSummary(); - + // If we have a Vulkan summary if (vulkanSummary != null) { // Extract Vulkan version from summary string? vulkanVersion = ExtractVulkanVersionFromSummary(vulkanSummary); - + // If we have a Vulkan version if (vulkanVersion != null) { @@ -61,11 +61,11 @@ public static SystemInfo Get() return vulkanVersion; } } - + // Return null if we failed to get the Vulkan version return null; } - + private static string? GetVulkanSummary() { // Note: on Linux, this requires `vulkan-tools` to be installed. (`sudo apt install vulkan-tools`) @@ -102,19 +102,19 @@ public static SystemInfo Get() // We have three ways of parsing the Vulkan version from the summary (output is a different between Windows and Linux) // For now, I have decided to go with the full version number, and leave it up to the user to parse it further if needed // I have left the other patterns in, in case we need them in the future - + // Output on linux : 4206847 (1.3.255) // Output on windows : 1.3.255 string pattern = @"apiVersion\s*=\s*([^\r\n]+)"; - + // Output on linux : 4206847 // Output on windows : 1.3.255 //string pattern = @"apiVersion\s*=\s*([\d\.]+)"; - + // Output on linux : 1.3.255 // Output on windows : 1.3.255 //string pattern = @"apiVersion\s*=\s*(?:\d+\s*)?(?:\(\s*)?([\d]+\.[\d]+\.[\d]+)(?:\s*\))?"; - + // Create a Regex object to match the pattern Regex regex = new Regex(pattern); @@ -158,24 +158,30 @@ private static int GetCudaMajorVersion() } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { + string? env_version = Environment.GetEnvironmentVariable("CUDA_VERSION"); + if (env_version is not null) + { + return ExtractMajorVersion(ref env_version); + } + // List of default cuda paths string[] defaultCudaPaths = [ "/usr/local/bin/cuda", "/usr/local/cuda", ]; - + // Loop through every default path to find the version foreach (var path in defaultCudaPaths) { // Attempt to get the version from the path version = GetCudaVersionFromPath(path); - + // If a CUDA version is found, break the loop if (!string.IsNullOrEmpty(version)) break; } - + if (string.IsNullOrEmpty(version)) { cudaPath = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH"); @@ -197,6 +203,11 @@ private static int GetCudaMajorVersion() if (string.IsNullOrEmpty(version)) return -1; + return ExtractMajorVersion(ref version); + } + + private static int ExtractMajorVersion(ref string version) + { version = version.Split('.')[0]; if (int.TryParse(version, out var majorVersion)) return majorVersion; From 81308419809560440d2e2fc031f043f8cf0dd7a9 Mon Sep 17 00:00:00 2001 From: Joey Spiker Date: Sat, 19 Jul 2025 10:26:31 -0400 Subject: [PATCH 2/3] clean-up white space -- fix for cuda version from env variable when working with nvidia/cuda containers --- LLama/Native/Load/SystemInfo.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/LLama/Native/Load/SystemInfo.cs b/LLama/Native/Load/SystemInfo.cs index 9e6d3e4fa..cc1a99ccb 100644 --- a/LLama/Native/Load/SystemInfo.cs +++ b/LLama/Native/Load/SystemInfo.cs @@ -41,7 +41,6 @@ public static SystemInfo Get() return new SystemInfo(platform, GetCudaMajorVersion(), GetVulkanVersion()); } - #region Vulkan version private static string? GetVulkanVersion() { @@ -65,7 +64,6 @@ public static SystemInfo Get() // Return null if we failed to get the Vulkan version return null; } - private static string? GetVulkanSummary() { // Note: on Linux, this requires `vulkan-tools` to be installed. (`sudo apt install vulkan-tools`) @@ -96,7 +94,6 @@ public static SystemInfo Get() return null; } } - static string? ExtractVulkanVersionFromSummary(string vulkanSummary) { // We have three ways of parsing the Vulkan version from the summary (output is a different between Windows and Linux) @@ -199,22 +196,18 @@ private static int GetCudaMajorVersion() } } } - if (string.IsNullOrEmpty(version)) return -1; return ExtractMajorVersion(ref version); } - private static int ExtractMajorVersion(ref string version) { version = version.Split('.')[0]; if (int.TryParse(version, out var majorVersion)) return majorVersion; - return -1; } - private static string GetCudaVersionFromPath(string cudaPath) { try @@ -237,7 +230,6 @@ private static string GetCudaVersionFromPath(string cudaPath) return string.Empty; } } - // Put it here to avoid calling NativeApi when getting the cuda version. [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] internal static extern int AddDllDirectory(string NewDirectory); @@ -245,4 +237,4 @@ private static string GetCudaVersionFromPath(string cudaPath) private const string cudaVersionFile = "version.json"; #endregion } -} +} \ No newline at end of file From e59bf81dd27138d3de1ad20ac6ebbfaa2102787c Mon Sep 17 00:00:00 2001 From: Joey Spiker Date: Sat, 19 Jul 2025 10:30:09 -0400 Subject: [PATCH 3/3] fix for cuda version from env variable when working with nvidia/cuda containers -- white space cleaup --- LLama/Native/Load/SystemInfo.cs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/LLama/Native/Load/SystemInfo.cs b/LLama/Native/Load/SystemInfo.cs index cc1a99ccb..0f48ce44f 100644 --- a/LLama/Native/Load/SystemInfo.cs +++ b/LLama/Native/Load/SystemInfo.cs @@ -46,13 +46,11 @@ public static SystemInfo Get() { // Get Vulkan Summary string? vulkanSummary = GetVulkanSummary(); - // If we have a Vulkan summary if (vulkanSummary != null) { // Extract Vulkan version from summary string? vulkanVersion = ExtractVulkanVersionFromSummary(vulkanSummary); - // If we have a Vulkan version if (vulkanVersion != null) { @@ -60,7 +58,6 @@ public static SystemInfo Get() return vulkanVersion; } } - // Return null if we failed to get the Vulkan version return null; } @@ -83,7 +80,6 @@ public static SystemInfo Get() } }; var (exitCode, output, error, ok) = process.SafeRun(TimeSpan.FromSeconds(12)); - // If ok return the output else return null return ok ? output : null; @@ -99,32 +95,25 @@ public static SystemInfo Get() // We have three ways of parsing the Vulkan version from the summary (output is a different between Windows and Linux) // For now, I have decided to go with the full version number, and leave it up to the user to parse it further if needed // I have left the other patterns in, in case we need them in the future - // Output on linux : 4206847 (1.3.255) // Output on windows : 1.3.255 string pattern = @"apiVersion\s*=\s*([^\r\n]+)"; - // Output on linux : 4206847 // Output on windows : 1.3.255 //string pattern = @"apiVersion\s*=\s*([\d\.]+)"; - // Output on linux : 1.3.255 // Output on windows : 1.3.255 //string pattern = @"apiVersion\s*=\s*(?:\d+\s*)?(?:\(\s*)?([\d]+\.[\d]+\.[\d]+)(?:\s*\))?"; - // Create a Regex object to match the pattern Regex regex = new Regex(pattern); - // Match the pattern in the input string Match match = regex.Match(vulkanSummary); - // If a match is found if (match.Success) { // Return the version number return match.Groups[1].Value; } - // Return null if no match is found return null; } @@ -142,15 +131,12 @@ private static int GetCudaMajorVersion() { return -1; } - //Ensuring cuda bin path is reachable. Especially for MAUI environment. string cudaBinPath = Path.Combine(cudaPath, "bin"); - if (Directory.Exists(cudaBinPath)) { AddDllDirectory(cudaBinPath); } - version = GetCudaVersionFromPath(cudaPath); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) @@ -160,14 +146,12 @@ private static int GetCudaMajorVersion() { return ExtractMajorVersion(ref env_version); } - // List of default cuda paths string[] defaultCudaPaths = [ "/usr/local/bin/cuda", "/usr/local/cuda", ]; - // Loop through every default path to find the version foreach (var path in defaultCudaPaths) { @@ -178,7 +162,6 @@ private static int GetCudaMajorVersion() if (!string.IsNullOrEmpty(version)) break; } - if (string.IsNullOrEmpty(version)) { cudaPath = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH"); @@ -233,7 +216,6 @@ private static string GetCudaVersionFromPath(string cudaPath) // Put it here to avoid calling NativeApi when getting the cuda version. [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] internal static extern int AddDllDirectory(string NewDirectory); - private const string cudaVersionFile = "version.json"; #endregion }