Add packaging step for llama.cpp

Signed-off-by: Ettore Di Giacinto <[email protected]>

Author: mudler

backend/Dockerfile.llama-cpp

@@ -195,22 +195,13 @@ if [ "${TARGETARCH}" = "arm64" ] || [ "${BUILD_TYPE}" = "hipblas" ]; then \
     fi  
 EOT
 
+
+# Copy libraries using a script to handle architecture differences
+RUN make -C /LocalAI/backend/cpp/llama-cpp package
+
+
 FROM scratch
 
-ARG BACKEND
-ENV BACKEND=${BACKEND}
-COPY --from=builder /LocalAI/backend/cpp/llama-cpp/llama-cpp-avx ./
-COPY --from=builder /LocalAI/backend/cpp/llama-cpp/llama-cpp-avx2 ./
-COPY --from=builder /LocalAI/backend/cpp/llama-cpp/llama-cpp-avx512 ./
-COPY --from=builder /LocalAI/backend/cpp/llama-cpp/llama-cpp-fallback ./
-COPY --from=builder /LocalAI/backend/cpp/llama-cpp/llama-cpp-grpc ./
-COPY --from=builder /LocalAI/backend/cpp/llama-cpp/llama-cpp-rpc-server ./
-COPY --from=builder /LocalAI/backend/cpp/llama-cpp/run.sh ./
-
-# copy ld.so from the base image in lib/ld.so
-COPY --from=builder /lib64/ld-linux-x86-64.so.2 ./lib/ld.so
-COPY --from=builder /lib/x86_64-linux-gnu/libc.so.6 ./lib/libc.so.6
-COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1 ./lib/libgcc_s.so.1
-COPY --from=builder /lib/x86_64-linux-gnu/libstdc++.so.6 ./lib/libstdc++.so.6
-COPY --from=builder /lib/x86_64-linux-gnu/libm.so.6 ./lib/libm.so.6
-COPY --from=builder /lib/x86_64-linux-gnu/libgomp.so.1 ./lib/libgomp.so.1
+
+# Copy all available binaries (the build process only creates the appropriate ones for the target architecture)
+COPY --from=builder /LocalAI/backend/cpp/llama-cpp/package/. ./

backend/cpp/llama-cpp/Makefile

@@ -138,6 +138,9 @@ rebuild:
 	rm -rf grpc-server
 	$(MAKE) grpc-server
 
+package:
+	bash package.sh
+
 purge:
 	rm -rf llama.cpp/build
 	rm -rf llama.cpp/tools/grpc-server

backend/cpp/llama-cpp/package.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Script to copy the appropriate libraries based on architecture
+# This script is used in the final stage of the Dockerfile
+
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+
+# Create lib directory
+mkdir -p $CURDIR/package/lib
+
+cp -rf $CURDIR/llama-cpp-* $CURDIR/package/
+
+# Detect architecture and copy appropriate libraries
+if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
+    # x86_64 architecture
+    echo "Detected x86_64 architecture, copying x86_64 libraries..."
+    cp /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
+    cp /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
+    # ARM64 architecture
+    echo "Detected ARM64 architecture, copying ARM64 libraries..."
+    cp /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
+    cp /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+else
+    echo "Error: Could not detect architecture"
+    exit 1
+fi
+
+echo "Packaging completed successfully"